diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..49463af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +.github +.forgejo +.vs +.vscode +venv*/ +__pycache__/ +*.pyc +archive/ +dotnet/ +tests/ +.pytest_cache/ +bin/temp/ +bin/ffmpeg +bin/ffmpeg.exe +bin/ffprobe +bin/TwitchDownloaderCLI +bin/TwitchDownloaderCLI.exe \ No newline at end of file diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..4ccc47f --- /dev/null +++ b/.env.development @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.env.production b/.env.production new file mode 100644 index 0000000..b1c4197 --- /dev/null +++ b/.env.production @@ -0,0 +1,16 @@ +TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest +TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive +TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production +TWITCH_ARCHIVE_ARCHIVE_BIND=./archive +TWITCH_ARCHIVE_CONFIG_BIND=./config +TWITCH_ARCHIVE_ARGS=-u vinesauce +TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce +TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf +PYTHONUNBUFFERED=1 +TZ=UTC +CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua +CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 +OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm +SENDER= +RECEIVER= +PASSWD= \ No newline at end of file diff --git a/.forgejo/workflows/publish-python-container.yml b/.forgejo/workflows/publish-python-container.yml new file mode 100644 index 0000000..831fa51 --- /dev/null +++ b/.forgejo/workflows/publish-python-container.yml @@ -0,0 +1,127 @@ +name: Publish Twitch Archive Container + +on: + push: + branches: + - master + - main + paths: + - docker/python.Dockerfile + - docker/entrypoint.sh + - docker-compose.yml + - docker-compose.override.yml + - requirements.txt + - twitch-archive.py + - run_chat_only.py + - modules/** + - .forgejo/workflows/publish-python-container.yml + workflow_dispatch: + +env: + REGISTRY: ${{ vars.FORGEJO_REGISTRY }} + IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }} + IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }} + BUILD_CONTEXT: . + DOCKERFILE_PATH: docker/python.Dockerfile + +jobs: + publish: + runs-on: docker + env: + DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate workflow variables + run: | + set -eu + if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi + if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi + if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi + if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi + + - name: Ensure Docker CLI exists + run: | + set -eu + if command -v docker >/dev/null 2>&1; then + docker --version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) DOCKER_ARCH="x86_64" ;; + aarch64|arm64) DOCKER_ARCH="aarch64" ;; + *) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;; + esac + + DOCKER_CLI_VERSION="27.5.1" + curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz + tar -xzf docker.tgz + mkdir -p "${HOME}/.local/bin" + mv docker/docker "${HOME}/.local/bin/docker" + chmod +x "${HOME}/.local/bin/docker" + echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}" + "${HOME}/.local/bin/docker" --version + + - name: Ensure Docker Buildx exists + run: | + set -eu + if docker buildx version >/dev/null 2>&1; then + docker buildx version + exit 0 + fi + + ARCH="$(uname -m)" + case "${ARCH}" in + x86_64) BUILDX_ARCH="amd64" ;; + aarch64|arm64) BUILDX_ARCH="arm64" ;; + *) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;; + esac + + BUILDX_VERSION="v0.21.1" + mkdir -p "${HOME}/.docker/cli-plugins" + curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx" + chmod +x "${HOME}/.docker/cli-plugins/docker-buildx" + docker buildx version + + - name: Check Docker daemon connectivity + run: | + set -eu + echo "Using DOCKER_HOST=${DOCKER_HOST}" + docker version + docker info >/dev/null + + - name: Create Buildx builder + run: | + set -eu + docker buildx rm forgejo-builder >/dev/null 2>&1 || true + docker buildx create --name forgejo-builder --driver docker-container --use + docker buildx inspect --bootstrap + + - name: Validate registry secrets + run: | + set -eu + if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi + if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi + + - name: Login to registry + run: | + set -eu + echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin + + - name: Build and push image + run: | + set -eu + IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}" + SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}" + BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}" + docker buildx build \ + --file "${DOCKERFILE_PATH}" \ + --tag "${SHA_TAG}" \ + --tag "${BRANCH_TAG}" \ + --tag "${IMAGE_REF}:latest" \ + --push \ + "${BUILD_CONTEXT}" \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 8e459b5..f887d97 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,22 @@ +# Git LFS tracking for large media and binaries + +# Binaries in bin/ +bin/* filter=lfs diff=lfs merge=lfs -text +bin/TwitchDownloaderCLI* filter=lfs diff=lfs merge=lfs -text +bin/ffmpeg filter=lfs diff=lfs merge=lfs -text +# Video files under archive (raw/ and common video extensions) +archive/**/video/** filter=lfs diff=lfs merge=lfs -text +archive/**/video/raw/** filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.mkv filter=lfs diff=lfs merge=lfs -text +*.mov filter=lfs diff=lfs merge=lfs -text +# Large archives and executables +*.zip filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.exe filter=lfs diff=lfs merge=lfs -text +*.dll filter=lfs diff=lfs merge=lfs -text # Auto detect text files and perform LF normalization * text=auto -bin/ffmpeg filter=lfs diff=lfs merge=lfs -text bin/ffmpeg.exe filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI.exe filter=lfs diff=lfs merge=lfs -text diff --git a/.github/instructions/blazor.instructions.md b/.github/instructions/blazor.instructions.md new file mode 100644 index 0000000..4e88cc0 --- /dev/null +++ b/.github/instructions/blazor.instructions.md @@ -0,0 +1,77 @@ +--- +description: 'Blazor component and application patterns' +applyTo: '**/*.razor, **/*.razor.cs, **/*.razor.css' +--- + +## Blazor Code Style and Structure + +- Write idiomatic and efficient Blazor and C# code. +- Follow .NET and Blazor conventions. +- Use Razor Components appropriately for component-based UI development. +- Prefer inline functions for smaller components but separate complex logic into code-behind or service classes. +- Async/await should be used where applicable to ensure non-blocking UI operations. + +## Naming Conventions + +- Follow PascalCase for component names, method names, and public members. +- Use camelCase for private fields and local variables. +- Prefix interface names with "I" (e.g., IUserService). + +## Blazor and .NET Specific Guidelines + +- Utilize Blazor's built-in features for component lifecycle (e.g., OnInitializedAsync, OnParametersSetAsync). +- Use data binding effectively with @bind. +- Leverage Dependency Injection for services in Blazor. +- Structure Blazor components and services following Separation of Concerns. +- Always use the latest version C#, currently C# 13 features like record types, pattern matching, and global usings. + +## Error Handling and Validation + +- Implement proper error handling for Blazor pages and API calls. +- Use logging for error tracking in the backend and consider capturing UI-level errors in Blazor with tools like ErrorBoundary. +- Implement validation using FluentValidation or DataAnnotations in forms. + +## Blazor API and Performance Optimization + +- Utilize Blazor server-side or WebAssembly optimally based on the project requirements. +- Use asynchronous methods (async/await) for API calls or UI actions that could block the main thread. +- Optimize Razor components by reducing unnecessary renders and using StateHasChanged() efficiently. +- Minimize the component render tree by avoiding re-renders unless necessary, using ShouldRender() where appropriate. +- Use EventCallbacks for handling user interactions efficiently, passing only minimal data when triggering events. + +## Caching Strategies + +- Implement in-memory caching for frequently used data, especially for Blazor Server apps. Use IMemoryCache for lightweight caching solutions. +- For Blazor WebAssembly, utilize localStorage or sessionStorage to cache application state between user sessions. +- Consider Distributed Cache strategies (like Redis or SQL Server Cache) for larger applications that need shared state across multiple users or clients. +- Cache API calls by storing responses to avoid redundant calls when data is unlikely to change, thus improving the user experience. + +## State Management Libraries + +- Use Blazor's built-in Cascading Parameters and EventCallbacks for basic state sharing across components. +- Implement advanced state management solutions using libraries like Fluxor or BlazorState when the application grows in complexity. +- For client-side state persistence in Blazor WebAssembly, consider using Blazored.LocalStorage or Blazored.SessionStorage to maintain state between page reloads. +- For server-side Blazor, use Scoped Services and the StateContainer pattern to manage state within user sessions while minimizing re-renders. + +## API Design and Integration + +- Use HttpClient or other appropriate services to communicate with external APIs or your own backend. +- Implement error handling for API calls using try-catch and provide proper user feedback in the UI. + +## Testing and Debugging in Visual Studio + +- All unit testing and integration testing should be done in Visual Studio. +- Test Blazor components and services using xUnit, NUnit, or MSTest. +- Use Moq or NSubstitute for mocking dependencies during tests. +- Debug Blazor UI issues using browser developer tools and Visual Studio's debugging tools for backend and server-side issues. +- For performance profiling and optimization, rely on Visual Studio's diagnostics tools. + +## Security and Authentication + +- Implement Authentication and Authorization in the Blazor app where necessary using ASP.NET Identity or JWT tokens for API authentication. +- Use HTTPS for all web communication and ensure proper CORS policies are implemented. + +## API Documentation and Swagger + +- Use Swagger/OpenAPI for API documentation for your backend API services. +- Ensure XML documentation for models and API methods for enhancing Swagger documentation. diff --git a/.gitignore b/.gitignore index b1a3beb..49c4387 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ config/global.json # Streamer-specific configurations (personal settings) config/streamers/*.json +config/rclone.conf +config/rclone.conf.* # Python cache __pycache__/ @@ -21,4 +23,49 @@ venv3/** .gitignore bin/** \n+# Ignore any virtual environment directories starting with 'venv' (venv, venv3, venv314, etc.) -venv*/ \ No newline at end of file +venv*/ +.vs/ProjectEvaluation/twitch-archive-2.metadata.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.projects.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.strings.v10.bin +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/CodeChunks.db +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/SemanticSymbols.db +.vs/Twitch-Archive-2/DesignTimeBuild/.dtbcache.v2 +.vs/Twitch-Archive-2/FileContentIndex/843065c8-d80f-4907-b0ae-6d010b3a5699.vsidx +.vs/Twitch-Archive-2/FileContentIndex/ef7e1a3c-80cd-4867-a9a8-2e5099471227.vsidx +.vs/Twitch-Archive-2/v18/.futdcache.v2 +.vs/Twitch-Archive-2/v18/.suo +.vs/Twitch-Archive-2/v18/DocumentLayout.backup.json +.vs/Twitch-Archive-2/v18/DocumentLayout.json +.vscode/settings.json + +# C# / Visual Studio +# Build Folders +bin/ +obj/ + +# Visual Studio files +*.user +*.suo +*.userprefs +*.csproj.user +*.pidb +*.pdb +*.cache +*.ilk +*.log +*.vspscc +*.vssscc + +# Test results and packages +TestResults/ +packages/ +*.nupkg + +# Database and backup +*.dbmdl +*.bak +*.backup +*.orig + +dotnet/.vs/** +.vs/** \ No newline at end of file diff --git a/README.md b/README.md index 40d7471..979e223 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,109 @@ # Twitch Archive Inspired by https://github.com/EnterGin/Auto-Stream-Recording-Twitch +## Git LFS + +This repository stores large media files (recorded video and some binaries). Use Git LFS to manage large objects. + +Quick setup (Windows): + +1. Install Git LFS: `git lfs install` +2. Ensure `.gitattributes` is committed (this repo includes one). +3. If you already have large files tracked by normal Git, migrate them: + +```powershell +git lfs install +git lfs track "*.mp4" "*.mkv" "bin/*" +git add .gitattributes +git add -A +git commit -m "Migrate large files to LFS" +git push origin main +``` + +Notes: +- Git LFS needs server-side support. If using GitHub, enable Git LFS on the remote and ensure you have sufficient bandwidth/storage quota. +- You can customize tracked patterns in `.gitattributes`. + Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. +## Docker + +This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow. + +Files: + +- `docker/python.Dockerfile`: production image for the Python archiver +- `docker-compose.yml`: deployment-oriented compose file +- `docker-compose.override.yml`: local development and testing override +- `.env.production`: production container and app environment template +- `.env.development`: development container and app environment template +- `dockerstart.bat`: Windows helper to run the container like the old batch launcher + +### Container layout + +- Mount your external archive folder to `/app/archive` +- Mount your external config folder to `/app/config` +- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path +- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically + +### Production deployment + +1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments. +2. Place your streamer JSON files and `rclone.conf` in the mounted config folder. +3. Start the container: + +```powershell +docker compose --env-file .env.production up -d +``` + +4. Follow logs: + +```powershell +docker compose --env-file .env.production logs -f twitch-archive +``` + +### Development and local testing + +The override compose file builds the image locally and mounts the repository for faster iteration. + +Start it with: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build +``` + +Run a one-off manual test for another streamer: + +```powershell +docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose +``` + +Or use the Windows helper: + +```powershell +.\dockerstart.bat vinesauce --verbose +``` + +That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. + +If the host has the NVIDIA Container Toolkit installed and you want FFmpeg/NVENC inside the container, use the optional NVIDIA override: + +```powershell +.\dockerrebuild.bat +.\dockerstart.bat --nvidia vinesauce --verbose +``` + +The image built by `.\dockerrebuild.bat` already includes the NVIDIA-capable FFmpeg/container toolchain. The optional [docker-compose.nvidia.yml](docker-compose.nvidia.yml) layer is only for runtime GPU passthrough: it requests `gpus: all` and sets `NVIDIA_VISIBLE_DEVICES` plus `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` for the container. + +On systems without NVIDIA support, keep using the normal command without `--nvidia`; the image still builds the same way, it just runs without GPU passthrough. + +### Healthcheck and smoke tests + +- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` +- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test` + +The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs. + ## ⚡ FFmpeg 8.0 Enhanced Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements! - **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs diff --git a/UpgradePlan.md b/UpgradePlan.md new file mode 100644 index 0000000..6654d3d --- /dev/null +++ b/UpgradePlan.md @@ -0,0 +1,214 @@ +Plan: C# .NET 10 Twitch Archive Rewrite +A complete port of the Python archiver to C# .NET 10 with Blazor Server UI, real-time process output, SQLite state tracking, full DI/service pattern, NLog logging, and a resilient recording engine. Placed under dotnet/ in the existing repo. + +Project Layout +Step 1 — Solution & Project scaffolding +Create the SLN and three projects: + +TwitchArchive.Core — classlib, targets net10.0 +TwitchArchive.Web — Blazor Server (blazorserver), targets net10.0 +TwitchArchive.Tests — xUnit, targets net10.0 +NuGet packages: + +Core: Microsoft.EntityFrameworkCore.Sqlite, Polly, NLog, NLog.Extensions.Logging +Web: all Core packages + Microsoft.AspNetCore.SignalR, NLog.Web.AspNetCore +Tests: xunit, Moq, coverlet.collector, Microsoft.EntityFrameworkCore.InMemory +Step 2 — Configuration models +Mirror the existing JSON schemas as C# POCOs with System.Text.Json attributes: + +GlobalConfig.cs — one property per key in config/global.json.example +StreamerConfig.cs — all fields nullable (override semantics), only Username and Enabled required +EffectiveConfig.cs — computed merge of global + per-streamer; exposes resolved values +AppSettings.cs — app-level settings (password hash, tool paths, .env secrets) +IConfigurationService / ConfigurationService: + +LoadGlobal() / SaveGlobal(GlobalConfig) +LoadStreamer(string username) / SaveStreamer(StreamerConfig) +GetAllStreamers(), GetEffectiveConfig(string username) (merge logic) +Reads/writes global.json and config/streamers/*.json — same files as Python +Step 3 — Infrastructure layer +TwitchApiClient (injectable, mockable): + +GetOAuthTokenAsync() — POST to https://id.twitch.tv/oauth2/token, caches token, refreshes on 401 +CheckStreamStatusAsync(string username) — GQL query for live stream + archiveVideo.id +GetLatestVodAsync(string username) — GQL query for most recent VOD +ValidateUsernameAsync(string username) — Helix /users endpoint +Credentials read from environment (CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN) +All methods return typed result objects, never throw on network errors — return Result (or OneOf) +HttpResiliencePolicy (Polly): + +Wraps HttpClient for TwitchApiClient +WaitAndRetryForever with exponential backoff starting at 15 s, doubling, capped at 10 minutes +Only applies to transient errors (5xx, timeout, HttpRequestException) — not 401/404 +Logged via NLog on each retry attempt +ProcessRunner (injectable + mockable for tests): + +RunAsync(ProcessRunOptions options, CancellationToken ct) → int exitCode +StartAsync(ProcessRunOptions options, CancellationToken ct) → IRunningProcess handle (for long-lived processes like streamlink) +Reads stdout and stderr line by line asynchronously +Reports each line to IProcessOutputStore (streamer + job context) +Forwards to NLog +ProcessRunOptions: FileName, Arguments, WorkingDirectory, RedirectOutput + +Step 4 — Core services (all behind interfaces) +IStreamMonitorService / StreamMonitorService + +Wraps TwitchApiClient +CheckIsLiveAsync(string username) → LiveStreamInfo? +GetLatestVodAsync(string username) → VodInfo? +IRecorderService / RecorderService + +StartRecordingAsync(string username, string quality, string outputPath, CancellationToken ct) → Task +Invokes streamlink via ProcessRunner +Passes --hls-live-restart, --stream-segment-threads, optional OAuth header +Returns when streamlink exits (either stream ended or ct cancelled) +IProcessorService / ProcessorService + +ProcessRawStreamAsync(string rawPath, string outputPath, EffectiveConfig cfg, CancellationToken ct) +Builds ffmpeg args: hwaccel, thread count, error recovery flags, faststart, copy codecs +MergeVideoChatAsync(string videoPath, string chatVideoPath, string outputPath, string layout, CancellationToken ct) +IDownloaderService / DownloaderService + +DownloadVodAsync(VodInfo vod, string outputPath, EffectiveConfig cfg, CancellationToken ct) → bool +Invokes TwitchDownloaderCLI videodownload +Chat download methods stubbed with NotImplementedException / commented structure; interface is defined now to keep the architecture clean +IUploadService / UploadService + +UploadAsync(string localRoot, IEnumerable relativeFilePaths, string rcloneDest, CancellationToken ct) → bool +Writes a temp files-from list, invokes rclone copy --files-from +Returns success/failure; preserves local files on failure +IFileManagerService / FileManagerService + +InitializeDirectories(string rootPath, string username) +GetPaths(string rootPath, string username, string filenameBase) → ArchivePaths record (all expected paths) +CleanRawFile(string path, bool cleanRaw) +DeleteLocalFiles(ArchivePaths paths, EffectiveConfig cfg) +GetUniquePath(string path) → adds numeric suffix if file exists +Step 5 — Recording resilience engine +RecoveryPolicy (POCO, unit-testable, no DI deps): + +Encodes a state machine with these states: + +State Meaning +Monitoring Normal polling at refresh interval +Recording streamlink subprocess active +FastReconnect Stream ended; checking every 10 s for up to 2 minutes +SlowReconnect Still not live after 2 min; checking every 60 s concurrently with post-processing +PostProcessing Confirmed ended; ffmpeg / VOD download / upload running +NetworkFault Twitch API unreachable; exponential back-off (30 s → capped at 10 min) +Transitions: + +Recording → streamlink exits → enter FastReconnect, record phase start time +FastReconnect → live confirmed → start new Recording (new filename/segment) +FastReconnect (2 min elapsed) → enter SlowReconnect + kick off PostProcessing concurrently +SlowReconnect → live confirmed → start new Recording +SlowReconnect / Monitoring → API call throws network error → enter NetworkFault +NetworkFault → successful API response → return to previous state (Monitoring or re-enter FastReconnect if we were mid-reconnect) +NetworkFault backoff: 30s * 2^attempt, capped at 600s +RecoveryPolicy is a pure class with a Tick(DateTime now, bool? isLive, bool networkError) method → returns RecoveryDecision (what to do next + sleep duration). Fully unit-testable with no async or DI. + +StreamWorker : BackgroundService + +One instance per enabled streamer +Holds RecoveryPolicy instance +Main loop: evaluate policy decision → execute the corresponding service call → loop +Started/stopped by StreamWorkerManager +Writes job records to SQLite on start/complete/fail +StreamWorkerManager + +StartWorker(string username), StopWorker(string username), RestartWorker(string username) +Called at app startup for all enabled streamers +Called from Web UI on enable/disable/config change +Workers stored in ConcurrentDictionary +Step 6 — Persistence (SQLite + EF Core) +ArchiveDbContext with three tables: + +StreamSessions: Id, StreamerUsername, TwitchStreamId, Title, StartedAt, EndedAt, Status (Recording/Processing/Uploading/Complete/Failed) + +ArchiveJobs: Id, SessionId, JobType (enum: RecordLive, ProcessLive, DownloadVod, ProcessVod, UploadCloud, DeleteLocal), Status, StartedAt, CompletedAt, FilePath, ErrorMessage + +StreamerStates: Username, IsMonitoring, LastCheckedAt, CurrentRecoveryState + +Migrations via EF Core CLI. ISessionRepository / IJobRepository interfaces for testability with in-memory EF provider in tests. + +Step 7 — Process output streaming +IProcessOutputStore: + +AppendLine(string streamerId, Guid jobId, string line, bool isError) +GetRecentLines(string streamerId, Guid jobId, int count = 500) → IReadOnlyList +In-memory circular buffer (1000 lines per job, last 20 jobs per streamer) +ProcessOutputHub : Hub (SignalR): + +Clients call SubscribeToStreamer(string username) → join group streamer:{username} +Clients call SubscribeToJob(Guid jobId) → join group job:{jobId} +Server pushes ReceiveLine(OutputLine line) from ProcessRunner via IHubContext +On subscribe: server immediately sends buffered lines from IProcessOutputStore +Step 8 — Blazor Server Web UI +Authentication: Cookie-based single-password auth via ASP.NET Core minimal auth middleware. Password stored as BCrypt hash in AppSettings. Login.razor page at /login. Protected routes with [Authorize]. + +Pages & Components: + +Dashboard.razor (/) — grid of all configured streamers showing: username, live/offline badge, current recovery state, last recorded session, quick Start/Stop monitoring toggle + +StreamerDetail.razor (/streamer/{username}) — live status, current job pipeline steps (record → process → upload with progress), ProcessOutputConsole.razor showing real-time terminal output via SignalR + +ProcessOutputConsole.razor — reusable Blazor component; subscribes to SignalR on mount, renders an auto-scrolling
 with colored output (stdout = white, stderr = orange/red), handles reconnect
+
+Sessions.razor (/sessions) — paginated list of past archive sessions with job statuses and expandable per-job output
+
+GlobalConfig.razor (/config/global) — EditForm bound to GlobalConfig model with data annotations validation, Save button calls IConfigurationService.SaveGlobal()
+
+StreamerConfig.razor (/config/{username}) — similar form for per-streamer overrides; each field has a nullable toggle (inherit from global vs override)
+
+AddStreamer.razor (/config/new) — minimal form: username + enabled; creates new config/streamers/{username}.json
+
+AppSettings.razor (/settings) — tool paths (streamlink, ffmpeg, TwitchDownloaderCLI, rclone), change password
+
+Step 9 — NLog configuration
+nlog.config (XML): two targets:
+
+Console (colored, with level formatting)
+File rolling (logs/archive-${shortdate}.log, keep 30 days)
+Log structured context: StreamerUsername, JobId, JobType as NLog ScopeContext properties. Service methods open a scope via ILogger.BeginScope(...).
+
+Step 10 — Docker
+Dockerfile (multi-stage):
+
+Build stage: mcr.microsoft.com/dotnet/sdk:10.0
+Runtime stage: mcr.microsoft.com/dotnet/aspnet:10.0 (Linux)
+Install ffmpeg, streamlink (via pip), download TwitchDownloaderCLI binary for linux-x64
+rclone installed via shell script or apt
+Expose port 8080
+ENTRYPOINT ["dotnet", "TwitchArchive.Web.dll"]
+docker-compose.yml:
+
+Volume mounts: ./config:/app/config, ./archive:/app/archive, ./logs:/app/logs
+Environment variables: CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN
+Windows dev: run directly with dotnet run; tool paths auto-detected (Windows vs Linux) via RuntimeInformation.IsOSPlatform(OSPlatform.Windows) in ToolPathResolver
+
+Step 11 — Unit tests
+TwitchArchive.Tests covers:
+
+RecoveryPolicyTests — state machine transitions, timing phases, network fault backoff; pure synchronous tests
+ConfigurationServiceTests — JSON load/save/merge with temp files
+TwitchApiClientTests — mocked HttpMessageHandler; OAuth, GQL queries, 401 refresh, network errors
+FileManagerServiceTests — path generation, directory creation with temp directories
+DownloaderServiceTests / RecorderServiceTests — mocked ProcessRunner; verify correct CLI arguments
+UploadServiceTests — mocked ProcessRunner; verify rclone argument construction
+SessionRepositoryTests — EF Core in-memory provider
+EffectiveConfigTests — global + streamer override merge logic
+Verification
+dotnet build dotnet/TwitchArchive.sln — zero warnings, zero errors
+dotnet test dotnet/TwitchArchive.Tests/ — all tests green
+docker compose up --build in the dotnet/ folder → app reachable at http://localhost:8080
+Manual: add a test streamer config, enable monitoring, confirm it polls and records a live stream, runs ffmpeg, uploads via rclone
+Resilience manual test: kill network during recording → verify FastReconnect phase kicks in, resumes after connectivity restored
+Decisions
+
+Blazor Server chosen for real-time terminal output without a separate API; no WASM needed
+streamlink for live + TwitchDownloaderCLI for VOD (same split as Python; streamlink gives better live resilience)
+Simple BCrypt password auth (not full Identity — this is a single-user tool)
+RecoveryPolicy as a pure POCO state machine keeps the resilience logic fully unit-testable without async/mocking
+Polly WaitAndRetryForever on HttpClient handles persistent network failure independently of the application-level state machine; they are complementary — Polly handles individual HTTP call retries, RecoveryPolicy handles the overall workflow state
+Chat download service interface is defined in Step 4 but methods are stubbed — adding implementation later requires only filling in DownloaderService without touching any other layer
+Config files remain in the same format/location so the Python and C# versions can share the same config directory
\ No newline at end of file
diff --git a/UpgradePlan2.md b/UpgradePlan2.md
new file mode 100644
index 0000000..cc4ffbd
--- /dev/null
+++ b/UpgradePlan2.md
@@ -0,0 +1,78 @@
+
+
+
@Body
``` Add a top bar (`
`) with the app title "Twitch Archive" and a hamburger toggle ` + + + @if (showAdvanced) + { +
+

Advanced per-streamer defaults

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ } + +
+ +
+ + +@code { + private TwitchArchive.Core.Config.StreamerConfig model = new() { Enabled = true }; + private bool showAdvanced = false; + + // local fields for binding nullable/global override values + private bool downloadVOD = true; + private bool downloadCHAT = true; + private bool downloadLiveCHAT = true; + private bool mergeVideoChat = false; + private int? vodTimeout; + private bool uploadPreMergeVideo = true; + private bool uploadMergedVideo = true; + private bool uploadChatVideo = false; + private bool deleteFiles = false; + private bool onlyRaw = false; + private bool cleanRaw = true; + private int? hlsSegments; + private int? hlsSegmentsVOD; + private bool streamlinkTtvlol = false; + private string? ffmpegHwaccel; + private int? ffmpegThreads; + private string? ffmpegAudioCodec; + private int? ffmpegAudioSamplerate; + private string? ffmpegAudioBitrate; + private bool ffmpegErrorRecovery = true; + private bool ffmpegFaststart = true; + private bool ffmpegProgress = false; + + private void ToggleAdvanced() => showAdvanced = !showAdvanced; + + private void Save() + { + model.Username = model.Username?.Trim().ToLowerInvariant() ?? string.Empty; + if (string.IsNullOrWhiteSpace(model.Username)) return; + + // map local fields into nullable model properties + model.DownloadVOD = downloadVOD; + model.DownloadCHAT = downloadCHAT; + model.DownloadLiveCHAT = downloadLiveCHAT; + model.MergeVideoChat = mergeVideoChat; + model.VodTimeout = vodTimeout; + model.UploadPreMergeVideo = uploadPreMergeVideo; + model.UploadMergedVideo = uploadMergedVideo; + model.UploadChatVideo = uploadChatVideo; + model.DeleteFiles = deleteFiles; + model.OnlyRaw = onlyRaw; + model.CleanRaw = cleanRaw; + model.HlsSegments = hlsSegments; + model.HlsSegmentsVOD = hlsSegmentsVOD; + model.StreamlinkTtvlol = streamlinkTtvlol; + model.FfmpegHwaccel = ffmpegHwaccel; + model.FfmpegThreads = ffmpegThreads; + model.FfmpegAudioCodec = ffmpegAudioCodec; + model.FfmpegAudioSamplerate = ffmpegAudioSamplerate; + model.FfmpegAudioBitrate = ffmpegAudioBitrate; + model.FfmpegErrorRecovery = ffmpegErrorRecovery; + model.FfmpegFaststart = ffmpegFaststart; + model.FfmpegProgress = ffmpegProgress; + + ConfigService.SaveStreamer(model); + Nav.NavigateTo($"/config/{model.Username}"); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor new file mode 100644 index 0000000..a666c0b --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor @@ -0,0 +1,230 @@ +@page "/settings" +@using System.Text.Json +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService +@inject TwitchArchive.Web.Services.IAuthService Auth + +

Settings

+ +@if (saved) +{ +
Saved.
+} + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + @if (showDefaults) + { + + +
+

Defaults

+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+ } +
+ +
+ +
+ +

Change Password

+@if (!string.IsNullOrEmpty(pwError)) +{ +
@pwError
+} +
+ + + + +
+ +@code { + private TwitchArchive.Core.Config.GlobalConfig globalModel = new(); + private bool saved = false; + private bool showDefaults = false; + private string currentPw = string.Empty; + private string newPw = string.Empty; + private string confirmPw = string.Empty; + private string pwError = string.Empty; + + protected override void OnInitialized() + { + Load(); + } + + private void Load() + { + try + { + globalModel = ConfigService.LoadGlobal() ?? new TwitchArchive.Core.Config.GlobalConfig(); + } + catch { globalModel = new TwitchArchive.Core.Config.GlobalConfig(); } + } + + private void SaveGlobal() + { + try + { + ConfigService.SaveGlobal(globalModel); + saved = true; + // optionally notify auth or other services + } + catch { } + } + + private void ToggleDefaults() => showDefaults = !showDefaults; + + private void ChangePassword() + { + pwError = string.Empty; + if (!Auth.ValidatePassword(currentPw)) { pwError = "Current password incorrect"; return; } + if (string.IsNullOrWhiteSpace(newPw)) { pwError = "New password required"; return; } + if (newPw != confirmPw) { pwError = "Passwords do not match"; return; } + var hash = BCrypt.Net.BCrypt.HashPassword(newPw); + Auth.SetPasswordHash(hash); + Auth.Refresh(); + saved = true; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor new file mode 100644 index 0000000..1ec6d41 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor @@ -0,0 +1,10 @@ +@page "/config/global" +@inject NavigationManager Nav + +@code { + protected override void OnInitialized() + { + // Consolidated settings are now at /settings + Nav.NavigateTo("/settings", true); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Index.razor b/dotnet/src/TwitchArchive.Web/Pages/Index.razor new file mode 100644 index 0000000..e25b8cd --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Index.razor @@ -0,0 +1,96 @@ +@page "/" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Web.Services.SessionCacheService SessionCache + +

Dashboard

+ +@if (streamers.Count == 0) +{ +
No streamers configured. Add one on the Add Streamer page.
+} + +
+ @foreach (var s in streamers) + { +
+
+ @s + Edit + @(WorkerManager.IsRunning(s) ? "Live" : "Offline") +
+
+
Last session: @(lastStarts.ContainsKey(s) ? lastStarts[s].ToLocalTime().ToString() : "-")
+
+ + +
+
+
+ } +
+ +@* Show global feedback when there are streamers but no recent sessions *@ +@if (streamers.Count > 0 && (lastStarts == null || lastStarts.Count == 0)) +{ +
No recent sessions found for configured streamers.
+} + +@code { + private List streamers = new(); + private Dictionary lastStarts = new(); + + private void OnCacheUpdatedHandler() + { + _ = InvokeAsync(() => { + lastStarts = SessionCache.GetSnapshot(); + StateHasChanged(); + }); + } + + protected override async Task OnInitializedAsync() + { + LoadStreamers(); + lastStarts = SessionCache.GetSnapshot(); + SessionCache.Updated += OnCacheUpdatedHandler; + } + + private void LoadStreamers() + { + // Try to find the config/streamers folder from the app content root and parent folders. + string? cfgDir = FindConfigStreamersFolder(); + if (!string.IsNullOrEmpty(cfgDir) && Directory.Exists(cfgDir)) + { + streamers = Directory.GetFiles(cfgDir, "*.json").Select(f => Path.GetFileNameWithoutExtension(f)).ToList(); + } + } + + private string? FindConfigStreamersFolder() + { + // Prefer ContentRoot if available, fall back to Environment.CurrentDirectory. + var start = AppContext.BaseDirectory ?? Environment.CurrentDirectory; + var dir = new DirectoryInfo(start); + for (int i = 0; i < 6 && dir != null; i++) + { + var candidate = Path.Combine(dir.FullName, "config", "streamers"); + if (Directory.Exists(candidate)) return candidate; + dir = dir.Parent; + } + + // final attempt: repo-root relative (use project parent heuristics) + var alt = Path.Combine(Environment.CurrentDirectory, "..", "..", "..", "config", "streamers"); + try { alt = Path.GetFullPath(alt); } catch { } + if (Directory.Exists(alt)) return alt; + return null; + } + + // Index reads from the singleton SessionCacheService; updates are pushed via the Updated event. + + private void Start(string u) { WorkerManager.StartWorker(u); } + private async Task Stop(string u) { await WorkerManager.StopWorkerAsync(u); } + + public async ValueTask DisposeAsync() + { + SessionCache.Updated -= OnCacheUpdatedHandler; + await Task.CompletedTask; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Login.razor b/dotnet/src/TwitchArchive.Web/Pages/Login.razor new file mode 100644 index 0000000..ff4f1da --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Login.razor @@ -0,0 +1,20 @@ +@page "/login" +@attribute [AllowAnonymous] +@using Microsoft.AspNetCore.Components + +

Login

+ +@if (error) +{ +
Invalid password
+} + +
+ + +
+ +@code { + [Parameter] + public bool error { get; set; } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Media.razor b/dotnet/src/TwitchArchive.Web/Pages/Media.razor new file mode 100644 index 0000000..a42c78c --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Media.razor @@ -0,0 +1,59 @@ +@page "/media" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService + +

Media Library

+ +@if (string.IsNullOrWhiteSpace(archiveRoot)) +{ +
Archive root is not configured. Set it on the Settings page.
+} +else if (!Directory.Exists(archiveRoot)) +{ +
Archive root '@archiveRoot' does not exist on disk.
+} +else +{ + @if (entries.Count == 0) + { +
No media files found in '@archiveRoot'.
+ } + else + { + @foreach (var kv in entries) + { +
+

@kv.Key

+
    + @foreach (var f in kv.Value) + { +
  • @f.Name - @((f.Length/1024.0/1024.0).ToString("0.00")) MB - @f.LastWriteTime.ToLocalTime()
  • + } +
+
+ } + } +} + +@code { + private string? archiveRoot; + private Dictionary> entries = new(); + + protected override void OnInitialized() + { + var g = ConfigService.LoadGlobal(); + archiveRoot = g?.ArchiveRoot; + if (!string.IsNullOrWhiteSpace(archiveRoot) && Directory.Exists(archiveRoot)) + { + var di = new DirectoryInfo(archiveRoot); + foreach (var dir in di.GetDirectories()) + { + try + { + var files = dir.GetFiles("*.mp4").Concat(dir.GetFiles("*.mkv")).Concat(dir.GetFiles("*.ts")).Concat(dir.GetFiles("*.flv")).OrderByDescending(f => f.LastWriteTime).ToList(); + if (files.Count > 0) entries[dir.Name] = files; + } + catch { } + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor new file mode 100644 index 0000000..4cd79de --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor @@ -0,0 +1,37 @@ +@page "/monitor" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Core.Services.IProcessOutputStore OutputStore +@using TwitchArchive.Web.Shared + +

Streamer Monitor

+ +
+ + + +
+ +
+ Status: @status +
+ + + +@code { + private string username = "hackerling"; + private string status = "idle"; + + private void Start() + { + if (string.IsNullOrWhiteSpace(username)) return; + WorkerManager.StartWorker(username); + status = WorkerManager.IsRunning(username) ? "running" : "starting"; + } + + private async Task Stop() + { + if (string.IsNullOrWhiteSpace(username)) return; + await WorkerManager.StopWorkerAsync(username); + status = WorkerManager.IsRunning(username) ? "running" : "stopped"; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor new file mode 100644 index 0000000..256da43 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor @@ -0,0 +1,73 @@ +@page "/sessions" +@using TwitchArchive.Core.Persistence.Models +@inject TwitchArchive.Core.Persistence.ISessionRepository SessionRepo + +

Sessions

+ + + + + + + + + @foreach (var s in sessions) + { + + + + + + + + + @if (expandedSession == s.Id) + { + + } + } + +
IdStreamerStartedEndedStatusJobs
@s.Id@s.StreamerUsername@s.StartedAt.ToLocalTime()@(s.EndedAt?.ToLocalTime().ToString() ?? "-")@s.Status
+
    + @if (jobs?.Any() ?? false) + { + @foreach (var j in jobs) + { +
  • @j.Id - @j.JobType - @j.Status - @j.StartedAt.ToLocalTime() - @(j.FilePath ?? "")
  • + } + } + else + { +
  • No jobs
  • + } +
+
+ +@code { + private List sessions = new(); + private List? jobs; + private long? expandedSession; + + protected override async Task OnInitializedAsync() + { + await Refresh(); + } + + private async Task Refresh() + { + sessions = await SessionRepo.GetRecentSessionsAsync(50); + StateHasChanged(); + } + + private async Task ToggleJobs(long sessionId) + { + if (expandedSession == sessionId) + { + expandedSession = null; + jobs = null; + return; + } + jobs = await SessionRepo.GetJobsForSessionAsync(sessionId); + expandedSession = sessionId; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor new file mode 100644 index 0000000..7c9b082 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -0,0 +1,228 @@ +@page "/config/{Username}" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService +@inject NavigationManager Nav + +

Streamer Config: @Username

+ + + + + @if (saved) + { +
Saved.
+ } +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ + @* Streamlink path is global-only; not configurable per-streamer *@ + +
+

Per-streamer settings

+
+ + +
+
+ + +
+
+ + +
+
+ + + + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + +
+ +@* Confirmation modal *@ +@if (showConfirm) +{ + +} + +@code { + [Parameter] + public string Username { get; set; } = string.Empty; + private TwitchArchive.Core.Config.StreamerConfig model = new(); + private TwitchArchive.Core.Config.GlobalConfig? global; + private bool saved = false; + private bool showConfirm = false; + + // local values for nullable per-streamer settings (bind safely) + private bool downloadVODVal; + private bool downloadCHATVal; + private bool downloadLiveCHATVal; + private bool mergeVideoChatVal; + private string mergeChatLayoutVal = "side-by-side"; + private int? vodTimeoutVal; + private bool deleteFilesVal; + private bool uploadPreMergeVideoVal; + private bool uploadMergedVideoVal; + private bool uploadChatVideoVal; + private bool onlyRawVal; + private bool cleanRawVal; + private bool uploadToCloudVal; + + protected override void OnInitialized() + { + global = ConfigService.LoadGlobal(); + var s = ConfigService.LoadStreamer(Username); + var isNew = s == null; + if (s != null) model = s; + // initialize local values from model or global defaults + downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; + downloadCHATVal = model.DownloadCHAT ?? global?.Defaults.DownloadCHAT ?? true; + downloadLiveCHATVal = model.DownloadLiveCHAT ?? global?.Defaults.DownloadLiveCHAT ?? true; + mergeVideoChatVal = model.MergeVideoChat ?? global?.Defaults.MergeVideoChat ?? false; + mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; + vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; + deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; + uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; + uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; + uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; + onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; + cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; + uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; + + // when creating a new streamer config, populate model with global defaults so + // the streamer config stores initial values and subsequent edits use streamer values + if (isNew) + { + model.Quality = model.Quality ?? global?.DefaultQuality; + model.UploadToCloud = uploadToCloudVal; + model.UploadDestination = model.UploadDestination ?? global?.UploadDestination; + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; + } + } + + private void Save() + { + model.Username = Username; + if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null; + // Upload to cloud + model.UploadToCloud = uploadToCloudVal; + // Ensure global-only settings are not stored per-streamer + model.StreamlinkPath = null; + model.HlsSegments = null; + model.HlsSegmentsVOD = null; + model.StreamlinkTtvlol = null; + model.FfmpegHwaccel = null; + model.FfmpegThreads = null; + model.FfmpegAudioBitrate = null; + model.FfmpegAudioCodec = null; + model.FfmpegAudioSamplerate = null; + model.FfmpegErrorRecovery = null; + model.FfmpegFaststart = null; + model.FfmpegProgress = null; + // Per-streamer values: always map local values into the model + model.DownloadVOD = downloadVODVal; + model.DownloadCHAT = downloadCHATVal; + model.DownloadLiveCHAT = downloadLiveCHATVal; + model.MergeVideoChat = mergeVideoChatVal; + model.MergeChatLayout = mergeChatLayoutVal; + model.VodTimeout = vodTimeoutVal; + model.DeleteFiles = deleteFilesVal; + model.UploadPreMergeVideo = uploadPreMergeVideoVal; + model.UploadMergedVideo = uploadMergedVideoVal; + model.UploadChatVideo = uploadChatVideoVal; + model.OnlyRaw = onlyRawVal; + model.CleanRaw = cleanRawVal; + ConfigService.SaveStreamer(model); + saved = true; + } + + private void ConfirmDelete() + { + try + { + ConfigService.DeleteStreamer(Username); + showConfirm = false; + Nav.NavigateTo("/"); + } + catch + { + // ignore + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor new file mode 100644 index 0000000..c9ac1e6 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor @@ -0,0 +1,23 @@ +@page "/streamer/{Username}" +@using TwitchArchive.Core.Workers +@inject StreamWorkerManager WorkerManager + +

Streamer: @Username

+ +
+ Status: @(WorkerManager.IsRunning(Username) ? "Live" : "Offline") + Edit Settings +
+ +
+
Record
+
Process
+
Upload
+
+ + + +@code { + [Parameter] + public string Username { get; set; } = string.Empty; +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml new file mode 100644 index 0000000..1463d9e --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml @@ -0,0 +1,19 @@ +@page "/" +@namespace TwitchArchive.Web.Pages +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers + + + + + + Twitch Archive + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/Program.cs b/dotnet/src/TwitchArchive.Web/Program.cs new file mode 100644 index 0000000..12ddccb --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Program.cs @@ -0,0 +1,105 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.EntityFrameworkCore; +using TwitchArchive.Core.Services; +using TwitchArchive.Core.Config; +using TwitchArchive.Core.Persistence; +using TwitchArchive.Web.Hubs; +using TwitchArchive.Web.Services; + +var builder = WebApplication.CreateBuilder(args); + +// Add services +builder.Services.AddRazorPages(); +builder.Services.AddServerSideBlazor(); +builder.Services.AddSignalR(); + +// Authentication +builder.Services.AddSingleton(); +builder.Services.AddAuthentication(options => +{ + options.DefaultScheme = Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme; +}) +.AddCookie(options => { options.LoginPath = "/login"; }); + +// Register Core services +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHttpClient(client => { /* base config if needed */ }); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// Configuration service for global + per-streamer JSON files in ./config +builder.Services.AddScoped(); + +// Broadcaster forwards output store events to the SignalR hub +builder.Services.AddSingleton(); + +// SQLite DB (file in app folder) +var conn = "Data Source=archive.db"; +// Provide a factory for creating DbContext instances for background work +builder.Services.AddDbContextFactory(opt => opt.UseSqlite(conn)); +// persistence +builder.Services.AddScoped(); + +// Session cache and background refresh +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + +var app = builder.Build(); + +if (!app.Environment.IsDevelopment()) +{ + app.UseExceptionHandler("/Error"); +} + +// Ensure DB schema exists (creates DB when missing) +using (var scope = app.Services.CreateScope()) +{ + var factory = scope.ServiceProvider.GetRequiredService>(); + try + { + using var db = factory.CreateDbContext(); + // Apply any pending EF migrations (creates/updates schema as needed) + db.Database.Migrate(); + } + catch { } +} + +app.UseStaticFiles(); +app.UseRouting(); + +app.UseAuthentication(); +app.UseAuthorization(); + +// login endpoint +app.MapPost("/auth/login", async (Microsoft.AspNetCore.Http.HttpContext http, TwitchArchive.Web.Services.IAuthService auth) => +{ + try + { + var form = await http.Request.ReadFormAsync(); + var pwd = form["password"].ToString(); + if (auth.ValidatePassword(pwd)) + { + var claims = new[] { new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "admin") }; + var id = new System.Security.Claims.ClaimsIdentity(claims, Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme); + var principal = new System.Security.Claims.ClaimsPrincipal(id); + await http.SignInAsync(Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme, principal); + http.Response.Redirect("/"); + return; + } + } + catch { } + http.Response.Redirect("/login?error=1"); +}); + +app.MapBlazorHub(); +app.MapHub("/processOutputHub"); +app.MapFallbackToPage("/_Host"); + +app.Run(); diff --git a/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json new file mode 100644 index 0000000..9c88f37 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "TwitchArchive.Web": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:64466;http://localhost:64467" + } + } +} \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs new file mode 100644 index 0000000..be83fc0 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs @@ -0,0 +1,75 @@ +using System; +using System.Linq; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using TwitchArchive.Core.Persistence; +using TwitchArchive.Core.Persistence.Models; + +namespace TwitchArchive.Web.Services +{ + public class AuthService : IAuthService + { + private readonly IDbContextFactory _dbFactory; + private readonly ILogger _log; + private string? _cachedHash; + + public AuthService(IDbContextFactory dbFactory, ILogger log) + { + _dbFactory = dbFactory ?? throw new ArgumentNullException(nameof(dbFactory)); + _log = log; + Refresh(); + } + + public void Refresh() + { + try + { + using var ctx = _dbFactory.CreateDbContext(); + var u = ctx.UserCredentials.AsNoTracking().FirstOrDefault(); + _cachedHash = u?.PasswordHash; + } + catch (Exception ex) + { + _log?.LogWarning(ex, "Failed to read password from database"); + _cachedHash = null; + } + } + + public bool ValidatePassword(string plain) + { + // If no password configured, allow access (default open) + if (string.IsNullOrWhiteSpace(_cachedHash)) return true; + try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _cachedHash); } + catch (Exception ex) + { + _log?.LogWarning(ex, "Password verification failed"); + return false; + } + } + + public void SetPasswordHash(string hash) + { + try + { + using var ctx = _dbFactory.CreateDbContext(); + var existing = ctx.UserCredentials.FirstOrDefault(); + if (existing == null) + { + existing = new UserCredential { PasswordHash = hash }; + ctx.UserCredentials.Add(existing); + } + else + { + existing.PasswordHash = hash; + ctx.UserCredentials.Update(existing); + } + ctx.SaveChanges(); + _cachedHash = hash; + } + catch (Exception ex) + { + _log?.LogError(ex, "Failed to save password to database"); + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs new file mode 100644 index 0000000..cd3b0fd --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; + +namespace TwitchArchive.Web.Services +{ + public interface IAuthService + { + bool ValidatePassword(string plain); + void Refresh(); + void SetPasswordHash(string hash); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs new file mode 100644 index 0000000..641ddf9 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs @@ -0,0 +1,35 @@ +using Microsoft.AspNetCore.SignalR; +using System; +using TwitchArchive.Core.Services; +using TwitchArchive.Web.Hubs; + +namespace TwitchArchive.Web.Services +{ + public class ProcessOutputBroadcaster : IDisposable + { + private readonly IProcessOutputStore _store; + private readonly IHubContext _hub; + + public ProcessOutputBroadcaster(IProcessOutputStore store, IHubContext hub) + { + _store = store; + _hub = hub; + _store.LineAppended += OnLineAppended; + } + + private void OnLineAppended(string streamer, OutputLine line) + { + try + { + // send to clients; clients should listen on "ReceiveLine" + _hub.Clients.Group(streamer).SendAsync("ReceiveLine", streamer, line); + } + catch { } + } + + public void Dispose() + { + _store.LineAppended -= OnLineAppended; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs new file mode 100644 index 0000000..abe7855 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using TwitchArchive.Core.Persistence.Models; + +namespace TwitchArchive.Web.Services +{ + public class SessionCacheService + { + private readonly object _lock = new(); + private Dictionary _snapshot = new(); + + public event Action? Updated; + + public void Update(IEnumerable sessions) + { + var next = new Dictionary(); + foreach (var s in sessions) + { + if (!next.ContainsKey(s.StreamerUsername)) next[s.StreamerUsername] = s.StartedAt; + } + + lock (_lock) + { + _snapshot = next; + } + + Updated?.Invoke(); + } + + public Dictionary GetSnapshot() + { + lock (_lock) + { + return new Dictionary(_snapshot); + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs new file mode 100644 index 0000000..93523b8 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using TwitchArchive.Core.Persistence; + +namespace TwitchArchive.Web.Services +{ + public class SessionRefreshHostedService : BackgroundService + { + private readonly IServiceScopeFactory _scopeFactory; + private readonly SessionCacheService _cache; + private readonly ILogger _logger; + private readonly TimeSpan _interval; + + public SessionRefreshHostedService(IServiceScopeFactory scopeFactory, SessionCacheService cache, ILogger logger) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _interval = TimeSpan.FromSeconds(10); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_interval); + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + using var scope = _scopeFactory.CreateScope(); + var repo = scope.ServiceProvider.GetRequiredService(); + var sessions = await repo.GetRecentSessionsAsync(200, stoppingToken).ConfigureAwait(false); + _cache.Update(sessions); + } + catch (OperationCanceledException) { break; } + catch (Exception ex) + { + _logger.LogError(ex, "Error while refreshing sessions"); + } + } + } + catch (OperationCanceledException) { } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor new file mode 100644 index 0000000..cf6d00c --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor @@ -0,0 +1,31 @@ +@inherits LayoutComponentBase +
+
+ +

Twitch Archive

+ +
+
+
+ +
+
@Body
+
+
+ +@code { + bool sidebarCollapsed; + void ToggleSidebar() => sidebarCollapsed = !sidebarCollapsed; +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor new file mode 100644 index 0000000..f5c2c76 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor @@ -0,0 +1,93 @@ +@using TwitchArchive.Core.Services +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.SignalR.Client +@using System.Linq +@inject IProcessOutputStore OutputStore +@inject NavigationManager Navigation + +@inherits ComponentBase +
+ @foreach (var line in lines) + { +
@line.TimestampUtc.ToLocalTime().ToString("HH:mm:ss") - @line.Line
+ } +
+ +@code { + [Parameter] + public string Streamer { get; set; } = string.Empty; + + private List lines = new(); + private HubConnection? hubConnection; + private string? _currentGroup; + + protected override async Task OnInitializedAsync() + { + OutputStore.LineAppended += OnLineAppended; + + hubConnection = new HubConnectionBuilder() + .WithUrl(Navigation.ToAbsoluteUri("/processOutputHub")) + .WithAutomaticReconnect() + .Build(); + + hubConnection.On("ReceiveLine", (streamer, line) => + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + // avoid duplicates + if (lines.Any(l => l.TimestampUtc == line.TimestampUtc && l.Line == line.Line)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + }); + + await hubConnection.StartAsync(); + if (!string.IsNullOrWhiteSpace(Streamer)) + { + await hubConnection.SendAsync("JoinStreamerGroup", Streamer); + _currentGroup = Streamer; + } + + await base.OnInitializedAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (hubConnection == null) return; + if (!string.IsNullOrWhiteSpace(_currentGroup) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); } catch { } + _currentGroup = null; + } + + if (!string.IsNullOrWhiteSpace(Streamer) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("JoinStreamerGroup", Streamer); _currentGroup = Streamer; } catch { } + } + + await base.OnParametersSetAsync(); + } + + private void OnLineAppended(string streamer, OutputLine line) + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + } + + public async ValueTask DisposeAsync() + { + OutputStore.LineAppended -= OnLineAppended; + if (hubConnection != null) + { + try + { + if (!string.IsNullOrWhiteSpace(_currentGroup)) await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); + } + catch { } + try { await hubConnection.StopAsync(); } catch { } + try { await hubConnection.DisposeAsync(); } catch { } + hubConnection = null; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj new file mode 100644 index 0000000..774299b --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/_Imports.razor b/dotnet/src/TwitchArchive.Web/_Imports.razor new file mode 100644 index 0000000..34b6706 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/_Imports.razor @@ -0,0 +1,10 @@ +@using System +@using System.Net.Http +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Authorization +@using TwitchArchive.Core +@using TwitchArchive.Core.Services +@using TwitchArchive.Web.Shared diff --git a/dotnet/src/TwitchArchive.Web/archive.db b/dotnet/src/TwitchArchive.Web/archive.db new file mode 100644 index 0000000..4e86411 Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm new file mode 100644 index 0000000..fd7d358 Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db-shm differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-wal b/dotnet/src/TwitchArchive.Web/archive.db-wal new file mode 100644 index 0000000..4596f14 Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db-wal differ diff --git a/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css new file mode 100644 index 0000000..5c247f8 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css @@ -0,0 +1,34 @@ +/* App layout styles for Twitch Archive */ +:root{ + --bg:#0f1720; --panel:#111322; --muted:#9aa4c5; --accent:#7dd3fc; --accent-2:#89b4fa; --card:#0b1220; +} +html,body { height:100%; margin:0; font-family:Segoe UI, Roboto, -apple-system, sans-serif; background:var(--bg); color:#e6eef8; } +.page { display:flex; min-height:100vh; } +.sidebar { width:220px; flex-shrink:0; background:var(--panel); color:var(--muted); overflow-y:auto; padding:1rem 0; box-shadow:2px 0 8px rgba(6,6,10,0.6); } +.sidebar.collapsed { width:66px; } +.brand { font-weight:600; padding:0 1rem 0.8rem 1rem; color:#fff; } +.nav-list { list-style:none; margin:0; padding:0; } +.nav-list li { margin:0.2rem 0; } +.nav-link { display:block; padding:0.6rem 1rem; color:var(--muted); text-decoration:none; border-left:4px solid transparent; } +.nav-link.active, .nav-link:hover { background:linear-gradient(90deg, rgba(255,255,255,0.02), transparent); color:#fff; border-left-color:var(--accent-2); } +.topbar { display:flex; align-items:center; background:var(--panel); padding:0.6rem 1rem; box-shadow:0 2px 6px rgba(0,0,0,0.4); } +.topbar .topbar-inner { display:flex; align-items:center; width:100%; } +.hamburger { font-size:1.2rem; margin-right:1rem; background:transparent; border:none; color:inherit; cursor:pointer; } +.title { margin:0; font-size:1.05rem; flex:1; } +.top-actions .action { color:var(--muted); text-decoration:none; margin-left:1rem; } +.main { flex:1; overflow-y:auto; padding:1.5rem; } +.container { max-width:1100px; margin:0 auto; } + +/* Cards and forms */ +.card { background:var(--card); padding:1rem; border-radius:8px; box-shadow:0 2px 8px rgba(2,6,23,0.6); margin-bottom:1rem; } +.card-header { display:flex; justify-content:space-between; align-items:center; font-weight:600; } +.card-body { margin-top:0.5rem; color:var(--muted); } +button { background:var(--accent); border:none; color:#002; padding:0.45rem 0.8rem; border-radius:6px; cursor:pointer; } +button:hover { opacity:0.95; } +input[type=password], input[type=text], input[type=number], .input, InputText { padding:0.5rem; border-radius:6px; border:1px solid rgba(255,255,255,0.06); background:transparent; color:inherit; } + +@media(max-width:768px) { + .sidebar { display:none; } + .topbar { display:flex; } +} + diff --git a/modules/constants.py b/modules/constants.py index 7f09e35..7aa10eb 100644 --- a/modules/constants.py +++ b/modules/constants.py @@ -32,6 +32,9 @@ DEFAULT_CONFIG = { 'mergeChatLayout': 'side-by-side', # Layout: 'side-by-side' or 'overlay' 'vodTimeout': 300, 'uploadCloud': True, + 'uploadPreMergeVideo': True, # Upload original videos before merging + 'uploadMergedVideo': True, # Upload merged videos (video + chat) + 'uploadChatVideo': False, # Upload standalone chat video 'deleteFiles': False, 'onlyRaw': False, 'cleanRaw': True, diff --git a/modules/downloader.py b/modules/downloader.py index 2507bd6..87fe30d 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -4,10 +4,13 @@ Includes fallback support for chat_downloader when VOD-based methods fail. """ import os +import sys import subprocess import json import threading import time +import socket +import re from typing import Dict, Any, Optional from colorama import Fore, Style @@ -38,11 +41,15 @@ class ContentDownloader: self.ffmpeg_path = ffmpeg_path self.quality = config.get('quality', 'best') self.hls_segments_vod = config.get('hls_segmentsVOD', 10) - self.download_vod = config.get('downloadVOD', True) - self.download_chat = config.get('downloadCHAT', True) - self.download_live_chat = config.get('downloadLiveCHAT', True) + self.download_vod_enabled = config.get('downloadVOD', True) + self.download_chat_enabled = config.get('downloadCHAT', True) + self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) + default_chat_font = 'Arial' if sys.platform.startswith('win') else 'DejaVu Sans' + self.chat_render_font = config.get('chat_render_font', default_chat_font) + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False # Initialize chat_downloader if available self.chat_downloader = None @@ -59,6 +66,11 @@ class ContentDownloader: self.chat_thread = None self.chat_thread_success = False self.chat_thread_error = None + + def reset_chat_render_status(self) -> None: + """Reset chat render tracking before a processing pass.""" + self.last_chat_render_attempted = False + self.last_chat_render_succeeded = False def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: """ @@ -71,7 +83,7 @@ class ContentDownloader: Returns: bool: True if download succeeded, False otherwise """ - if not self.download_vod: + if not self.download_vod_enabled: return False print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') @@ -188,7 +200,7 @@ class ContentDownloader: '-h', '1080', '--framerate', '30', '--outline', - '-f', 'Arial', + '-f', self.chat_render_font, '--font-size', '22', '--update-rate', '1.0', '--offline', @@ -213,6 +225,9 @@ class ContentDownloader: try: print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') + print(f'{Fore.CYAN}Using chat font: {self.chat_render_font}{Style.RESET_ALL}') + self.last_chat_render_attempted = True + self.last_chat_render_succeeded = False # Build complete command full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings @@ -247,6 +262,7 @@ class ContentDownloader: print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}') return False + self.last_chat_render_succeeded = True print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}') return True @@ -270,7 +286,7 @@ class ContentDownloader: Returns: bool: True if succeeded, False otherwise """ - if not self.download_chat: + if not self.download_chat_enabled: return False print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') @@ -296,7 +312,7 @@ class ContentDownloader: Returns: subprocess.Popen: The process handle, or None if failed to start """ - if not self.download_live_chat: + if not self.download_live_chat_enabled: return None print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') @@ -325,6 +341,118 @@ class ContentDownloader: except Exception as e: print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') return None + + def _download_live_chat_via_irc(self, username: str, json_path: str, + max_messages: Optional[int] = None, + timeout: Optional[float] = None, + shutdown_check: Optional[callable] = None, + stream_monitor = None, + verbose: bool = False) -> bool: + """ + Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail. + + This writes newline-delimited JSON objects with at least: timestamp (ms), + author (dict with `name`), and `message`. + """ + try: + sock = socket.socket() + sock.connect(('irc.chat.twitch.tv', 6667)) + sock.settimeout(1.0) + + # Request tags & capabilities + sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n') + sock.sendall(b'PASS SCHMOOPIIE\r\n') + sock.sendall(b'NICK justinfan67420\r\n') + sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8')) + + messages_written = 0 + start_time = time.time() + + # Open file for streaming newline-delimited JSON + os.makedirs(os.path.dirname(json_path), exist_ok=True) + with open(json_path, 'w', encoding='utf-8') as out_f: + buffer = '' + while True: + # Shutdown/timeouts + if shutdown_check and shutdown_check(): + break + if timeout and (time.time() - start_time) > timeout: + break + if stream_monitor: + try: + if not stream_monitor.is_user_live(): + break + except Exception: + pass + + try: + data = sock.recv(4096).decode('utf-8', 'ignore') + except socket.timeout: + continue + except Exception as e: + print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}') + break + + if not data: + continue + + buffer += data + lines = buffer.split('\r\n') + buffer = lines.pop() # remainder + + for line in lines: + if not line: + continue + # Respond to PINGs + if line.startswith('PING'): + try: + sock.sendall(b'PONG :tmi.twitch.tv\r\n') + except Exception: + pass + continue + + # Extract PRIVMSG lines + m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line) + if not m: + continue + + author = m.group(1) + msg_text = m.group(2) + timestamp_ms = int(time.time() * 1000) + + item = { + 'timestamp': timestamp_ms, + 'author': {'name': author}, + 'message': msg_text + } + + out_f.write(json.dumps(item, ensure_ascii=False) + '\n') + out_f.flush() + messages_written += 1 + + if verbose and (messages_written % 10 == 0): + print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}') + + if max_messages and messages_written >= max_messages: + break + + if max_messages and messages_written >= max_messages: + break + + sock.close() + + if messages_written > 0: + print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}') + return True + else: + print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}') + return False + + except Exception as e: + print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}') + import traceback + traceback.print_exc() + return False def wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str, timeout: int = 300) -> bool: @@ -365,6 +493,7 @@ class ContentDownloader: max_messages: Optional[int] = None, timeout: Optional[float] = None, shutdown_check: Optional[callable] = None, + stream_monitor = None, verbose: bool = False) -> bool: """ Download live chat using chat_downloader library as fallback. @@ -376,6 +505,7 @@ class ContentDownloader: max_messages: Maximum messages to download (None = unlimited) timeout: Stop after this many seconds (None = until stream ends) shutdown_check: Optional callback function that returns True when shutdown requested + stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat message previews Returns: @@ -386,9 +516,19 @@ class ContentDownloader: print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') return False - if not self.download_live_chat: + if not self.download_live_chat_enabled: print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') return False + + # If a stream monitor was provided, check that the user is currently live + if stream_monitor is not None: + try: + if not stream_monitor.is_user_live(): + print(f'{Fore.YELLOW}⚠ Stream is not live; skipping chat download{Style.RESET_ALL}') + return False + except Exception as e: + # If we couldn't determine live status, continue and let chat_downloader handle it + print(f'{Fore.YELLOW}⚠ Could not determine live status: {e} - proceeding with chat download{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Starting live chat download (chat_downloader)...{Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader library version: {ChatDownloader.__module__}{Style.RESET_ALL}') @@ -401,20 +541,53 @@ class ContentDownloader: print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}') - # Get chat messages + # Get chat messages with a small retry loop to handle transient GQL/network issues print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}') - chat = self.chat_downloader.get_chat( - stream_url, - message_types=['text_message'], # Basic text messages - output=json_path, - timeout=timeout, - max_messages=max_messages - ) + chat = None + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + chat = self.chat_downloader.get_chat( + stream_url, + message_types=['text_message'], # Basic text messages + output=json_path, + timeout=timeout, + max_messages=max_messages + ) + break + except Exception as e: + # Provide a clearer, user-facing message for common failures + print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}") + # On final attempt, dump traceback to help diagnose library internals + if attempt >= max_attempts: + print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}") + print(f"{Fore.YELLOW} Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}") + import traceback + traceback.print_exc() + # Try IRC fallback before giving up + print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}") + try: + return self._download_live_chat_via_irc(username, json_path, + max_messages=max_messages, + timeout=timeout, + shutdown_check=shutdown_check, + stream_monitor=stream_monitor, + verbose=verbose) + except Exception as fallback_err: + print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}") + traceback.print_exc() + return False + else: + time.sleep(1) + continue # The get_chat with output parameter writes to file automatically # We just need to iterate to trigger the download message_count = 0 - print(f'{Fore.CYAN}Receiving chat messages (press Ctrl+C to stop)...{Style.RESET_ALL}') + last_check_time = time.time() + check_interval = 10.0 # Check if stream is still live every 10 seconds + + print(f'{Fore.CYAN}Receiving chat messages (will stop when stream ends)...{Style.RESET_ALL}') try: for message in chat: # Check for shutdown request @@ -422,6 +595,19 @@ class ContentDownloader: print(f'\n{Fore.YELLOW}⚠ Chat download stopped by shutdown request{Style.RESET_ALL}') break + # Periodically check if stream is still live + current_time = time.time() + if stream_monitor and (current_time - last_check_time) >= check_interval: + last_check_time = current_time + try: + is_live = stream_monitor.is_user_live() + if not is_live: + print(f'\n{Fore.YELLOW}⚠ Stream ended, stopping chat download{Style.RESET_ALL}') + break + except Exception as check_error: + print(f'\n{Fore.YELLOW}⚠ Could not check stream status: {check_error}{Style.RESET_ALL}') + # Continue downloading to avoid false positives from API errors + message_count += 1 # Show progress every 100 messages @@ -467,6 +653,7 @@ class ContentDownloader: def start_chat_downloader_thread(self, username: str, json_path: str, shutdown_check: Optional[callable] = None, + stream_monitor = None, verbose: bool = False) -> threading.Thread: """ Start chat_downloader in a background thread. @@ -475,6 +662,7 @@ class ContentDownloader: username: Twitch username json_path: Path to save chat JSON shutdown_check: Callback to check for shutdown + stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat previews Returns: @@ -485,6 +673,7 @@ class ContentDownloader: self.chat_thread_success = self.download_live_chat_with_chat_downloader( username, json_path, shutdown_check=shutdown_check, + stream_monitor=stream_monitor, verbose=verbose ) except Exception as e: diff --git a/modules/file_manager.py b/modules/file_manager.py index 8848b8a..b024e8f 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -9,7 +9,7 @@ import subprocess from typing import List from colorama import Fore, Style -from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA +from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA, PREFIX_MERGED from .utils import get_bin_path @@ -28,6 +28,9 @@ class FileManager: self.root_path = pathlib.Path(root_path) self.username = username self.upload_cloud = config.get('uploadCloud', True) + self.upload_pre_merge_video = config.get('uploadPreMergeVideo', True) + self.upload_merged_video = config.get('uploadMergedVideo', True) + self.upload_chat_video = config.get('uploadChatVideo', True) self.delete_files = config.get('deleteFiles', False) self.clean_raw = config.get('cleanRaw', True) self.download_vod = config.get('downloadVOD', True) @@ -42,6 +45,128 @@ class FileManager: self.chat_mp4_path = self.root_path / username / "chat" self.metadata_path = self.root_path / username / "metadata" self.log_file = self.root_path / ".log" + + def _to_rclone_relative_path(self, *parts: str) -> str: + """Build a POSIX-style relative path for rclone --files-from.""" + return pathlib.PurePosixPath(*parts).as_posix() + + def _build_upload_relative_paths(self, filename_base: str) -> List[str]: + """Build the candidate upload list relative to root_path for rclone.""" + files_to_upload: List[str] = [ + self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"), + self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json") + ] + + if self.upload_pre_merge_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_merged_video: + files_to_upload.extend([ + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), + self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") + ]) + + if self.upload_chat_video: + files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) + + return files_to_upload + + def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]: + """Filter candidate upload paths to the files that actually exist.""" + existing_paths: List[str] = [] + for relative_path in relative_paths: + if (self.root_path / pathlib.PurePosixPath(relative_path)).exists(): + existing_paths.append(relative_path) + return existing_paths + + def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool: + """Run rclone copy for a set of paths relative to root_path.""" + existing_paths = self._get_existing_upload_relative_paths(relative_paths) + missing_paths = [path for path in relative_paths if path not in existing_paths] + + if not existing_paths: + print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + return False + + if missing_paths: + print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}') + for missing_path in missing_paths: + print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') + + print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}') + print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}') + print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}') + + bin_path = get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f: + f.write('\n'.join(existing_paths)) + f.write('\n') + + try: + cmd = [ + 'rclone', 'copy', + str(self.root_path.resolve()), + self.rclone_path, + '--files-from', upload_list_path, + '--progress' + ] + + print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}') + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + return proc.returncode == 0 + finally: + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + def run_rclone_smoke_test(self) -> bool: + """Create and upload a tiny metadata file to verify rclone output and configuration.""" + smoke_name = 'RCLONE_SMOKE_TEST' + smoke_relative_path = self._to_rclone_relative_path( + self.username, + 'metadata', + f"{PREFIX_METADATA}{smoke_name}.json" + ) + smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path) + + smoke_payload = { + 'type': 'rclone_smoke_test', + 'username': self.username + } + + smoke_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(smoke_file_path, 'w', encoding='utf-8') as f: + json.dump(smoke_payload, f, indent=2) + + print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}') + try: + result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test') + if result: + print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}') + return result + finally: + if smoke_file_path.exists(): + smoke_file_path.unlink() def initialize_directories(self) -> None: """Create all necessary directory structures.""" @@ -117,55 +242,25 @@ class FileManager: print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') if notification_callback: notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - # Create list of files to upload - bin_path = get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - - # Ensure temp directory exists - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - files_to_upload = [ - f"{PREFIX_LIVE}{filename_base}.ts", - f"{PREFIX_LIVE}{filename_base}.mp4", - f"{PREFIX_LIVE}{filename_base}.mp3", - f"{PREFIX_VOD}{filename_base}.ts", - f"{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_VOD}{filename_base}.mp3", - f"{PREFIX_METADATA}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.mp4" - ] - - with open(upload_list_path, 'w') as f: - f.write('\n'.join(files_to_upload)) - - # Run rclone + + files_to_upload = self._build_upload_relative_paths(filename_base) + try: - result = subprocess.call([ - 'rclone', 'copy', - str(self.root_path.resolve()), - self.rclone_path, - '--include-from', upload_list_path - ]) - - # Clean up upload list - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - if result == 0: + result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}') + + if result: print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') if notification_callback: notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') return True - else: - print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - if notification_callback: - notification_callback(f'✗ Upload Failed - {filename_base}', - f'Upload failed with code {result}. Files preserved locally.') - return False - + + print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✗ Upload Failed - {filename_base}', + 'Upload failed. Files preserved locally. Check rclone output above.') + return False + except Exception as e: print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') return False @@ -175,6 +270,8 @@ class FileManager: """ Delete local archive files after successful upload. + Only deletes files that were configured to be uploaded. + Args: filename_base: Base filename (without prefixes/extensions) live_raw_path: Path to live raw file @@ -191,14 +288,15 @@ class FileManager: files_to_delete: List[str] = [] - # Live files - if not self.clean_raw and os.path.exists(live_raw_path): - files_to_delete.append(live_raw_path) - if os.path.exists(live_proc_path): - files_to_delete.append(live_proc_path) + # Live files (only if pre-merge videos are uploaded) + if self.upload_pre_merge_video: + if not self.clean_raw and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) - # VOD files - if self.download_vod: + # VOD files (only if pre-merge videos are uploaded) + if self.download_vod and self.upload_pre_merge_video: vod_raw = self.raw_path / f"{PREFIX_VOD}{filename_base}.ts" vod_mp4 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp4" vod_mp3 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp3" @@ -210,17 +308,37 @@ class FileManager: if vod_mp3.exists(): files_to_delete.append(str(vod_mp3)) + # Merged video files (only if merged videos are uploaded) + if self.upload_merged_video: + merged_live_mp4 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp4" + merged_live_mp3 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp3" + merged_vod_mp4 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4" + merged_vod_mp3 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3" + + if merged_live_mp4.exists(): + files_to_delete.append(str(merged_live_mp4)) + if merged_live_mp3.exists(): + files_to_delete.append(str(merged_live_mp3)) + if merged_vod_mp4.exists(): + files_to_delete.append(str(merged_vod_mp4)) + if merged_vod_mp3.exists(): + files_to_delete.append(str(merged_vod_mp3)) + # Chat files if self.download_chat: chat_json = self.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json" - chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" + # Always delete JSON (it's always uploaded) if chat_json.exists(): files_to_delete.append(str(chat_json)) - if chat_mp4.exists(): - files_to_delete.append(str(chat_mp4)) + + # Only delete chat MP4 if chat videos are uploaded + if self.upload_chat_video: + chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" + if chat_mp4.exists(): + files_to_delete.append(str(chat_mp4)) - # Metadata files + # Metadata files (always uploaded) if self.download_metadata: metadata = self.metadata_path / f"{PREFIX_METADATA}{filename_base}.json" if metadata.exists(): diff --git a/modules/processor.py b/modules/processor.py index b1cae71..58bd47f 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -6,7 +6,7 @@ import os import subprocess from colorama import Fore, Style -from .utils import detect_hardware_acceleration, get_hwaccel_encoder +from .utils import detect_hardware_acceleration, get_hwaccel_encoder, resolve_hwaccel_type class StreamProcessor: @@ -36,38 +36,80 @@ class StreamProcessor: config.get('ffmpeg_hwaccel', 'auto'), os_type ) + self.hwaccel_type = resolve_hwaccel_type(self.hwaccel_type, os_type) - def process_raw_stream(self, raw_path: str, output_path: str) -> None: + def process_raw_stream(self, raw_path: str, output_path: str) -> bool: """ Process raw .ts file into mp4/mp3 using ffmpeg. Args: raw_path: Path to the raw .ts file output_path: Path for the processed output file + + Returns: + bool: True when conversion succeeded, False otherwise """ if not os.path.exists(raw_path): print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') - return + return False if self.only_raw: print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') - return + return False print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') # Build ffmpeg command based on quality if self.quality == 'audio_only': - self._process_audio(raw_path, output_path) + result = self._process_audio(raw_path, output_path) else: - self._process_video(raw_path, output_path) + result = self._process_video(raw_path, output_path) - print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + if result: + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') + else: + print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}') + + return result - def _process_audio(self, raw_path: str, output_path: str) -> None: + def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool: + """Run FFmpeg while streaming its output to the terminal.""" + print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}') + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding='utf-8', + errors='replace' + ) + + if process.stdout: + for line in process.stdout: + print(line, end='') + + result = process.wait() + if result != 0: + print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}') + return False + + if not os.path.exists(output_path): + print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}') + return False + + if os.path.getsize(output_path) == 0: + print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}') + return False + + return True + + def _process_audio(self, raw_path: str, output_path: str) -> bool: """Process audio-only stream.""" # Audio-only conversion with modern AAC encoding cmd = [ self.ffmpeg_path, + '-y', '-i', raw_path, '-vn', # No video '-c:a', self.ffmpeg_audio_codec, @@ -85,14 +127,9 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) - def _process_video(self, raw_path: str, output_path: str) -> None: + def _process_video(self, raw_path: str, output_path: str) -> bool: """Process video stream.""" cmd = [ self.ffmpeg_path, @@ -135,12 +172,7 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - - # Run FFmpeg - if self.ffmpeg_progress: - subprocess.call(cmd) - else: - subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + return self._run_ffmpeg_command(cmd, output_path) def build_chat_output_args(self) -> str: """ diff --git a/modules/recorder.py b/modules/recorder.py index fa9c9d1..3518591 100644 --- a/modules/recorder.py +++ b/modules/recorder.py @@ -7,6 +7,8 @@ import subprocess from typing import Dict, Any, Optional from colorama import Fore, Style +from .utils import get_env_value + class StreamRecorder: """Handles live stream recording using streamlink.""" @@ -68,7 +70,7 @@ class StreamRecorder: print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') # Add authentication if available - oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") + oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="") if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py index 6c1607a..a6b89be 100644 --- a/modules/stream_monitor.py +++ b/modules/stream_monitor.py @@ -9,6 +9,7 @@ import requests from colorama import Fore, Style from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID +from .utils import get_env_value class StreamMonitor: @@ -40,7 +41,9 @@ class StreamMonitor: return self._oauth_token try: - url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" + client_id = get_env_value('CLIENT-ID', 'CLIENT_ID') + client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials" response = requests.post(url, timeout=15) response.raise_for_status() self._oauth_token = response.json()['access_token'] @@ -69,7 +72,7 @@ class StreamMonitor: url = f'{TWITCH_API_URL}/users?login={self.username}' headers = { "Authorization": f"Bearer {self.get_oauth_token()}", - "Client-ID": os.getenv('CLIENT-ID') + "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID') } response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() @@ -115,6 +118,34 @@ class StreamMonitor: print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') sys.exit(1) + def is_user_live(self) -> bool: + """ + Check if the configured user is currently live. + + Returns: + bool: True if user is live, False if offline + + Raises: + Exception: If API request fails (caller should handle) + """ + query = f'query{{user(login: "{self.username}") {{stream{{id title}}}}}}' + + try: + response = requests.post( + TWITCH_GQL_URL, + json={'query': query}, + headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, + timeout=15 + ) + response.raise_for_status() + data = response.json() + stream_data = data.get('data', {}).get('user', {}).get('stream') + return stream_data is not None + + except requests.exceptions.RequestException as e: + # Don't exit, let caller handle this + raise Exception(f"Failed to check if user is live: {str(e)}") + def get_latest_vod(self) -> Optional[Dict[str, Any]]: """ Get the most recent VOD for the configured user. diff --git a/modules/utils.py b/modules/utils.py index 280afa8..4e3b9b1 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -4,6 +4,7 @@ Utility functions and helpers for Twitch Archive. import os import sys +import shutil import pathlib import subprocess from typing import Optional @@ -35,6 +36,15 @@ def get_bin_path() -> str: return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") +def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]: + """Return the first non-empty environment variable from the provided names.""" + for name in names: + value = os.getenv(name) + if value not in (None, ""): + return value + return default + + def get_ffmpeg_executable(os_type: str) -> str: """ Get the platform-specific ffmpeg executable path. @@ -48,6 +58,11 @@ def get_ffmpeg_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'ffmpeg.exe') + + system_ffmpeg = shutil.which('ffmpeg') + if system_ffmpeg: + return system_ffmpeg + return os.path.join(bin_path, 'ffmpeg') @@ -64,6 +79,11 @@ def get_twitch_downloader_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') + + system_twitch_downloader = shutil.which('TwitchDownloaderCLI') + if system_twitch_downloader: + return system_twitch_downloader + return os.path.join(bin_path, 'TwitchDownloaderCLI') @@ -164,6 +184,24 @@ def verify_streamlink() -> bool: return False +def verify_rclone() -> bool: + """Verify that rclone is available on PATH.""" + try: + result = subprocess.run(['rclone', 'version'], + capture_output=True, + text=True, + timeout=5) + if result.returncode == 0: + version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown' + print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}') + return True + raise FileNotFoundError() + except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): + print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}') + return False + + def verify_ffmpeg(os_type: str) -> bool: """ Verify that ffmpeg is available. @@ -231,20 +269,57 @@ def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[ if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: return hwaccel_config - # Auto-detect: try to determine available hardware + # Auto-detect: choose only hardware we can reasonably prove is present. if hwaccel_config == 'auto': - # On Windows, NVIDIA is most common - if os_type == 'windows': - # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide - return 'auto' - else: - # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA - # Let ffmpeg auto-detect - return 'auto' + if is_nvidia_runtime_available(): + return 'nvenc' + if is_vaapi_runtime_available(): + return 'vaapi' + return 'none' return None +def is_nvidia_runtime_available() -> bool: + """Return True when the current runtime appears to expose an NVIDIA GPU.""" + visible_devices = os.getenv('NVIDIA_VISIBLE_DEVICES', '').strip().lower() + if visible_devices in {'void', 'none'}: + return False + if visible_devices and visible_devices != 'all': + return True + + if shutil.which('nvidia-smi'): + return True + + return any( + os.path.exists(device_path) + for device_path in ('/dev/nvidiactl', '/dev/nvidia0', '/dev/nvidia-modeset') + ) + + +def is_vaapi_runtime_available() -> bool: + """Return True when Linux VAAPI render nodes are present.""" + return any( + os.path.exists(device_path) + for device_path in ('/dev/dri/renderD128', '/dev/dri/card0') + ) + + +def resolve_hwaccel_type(hwaccel_type: Optional[str], os_type: str) -> Optional[str]: + """Return a safe hardware acceleration choice for the current runtime.""" + if hwaccel_type in (None, 'none'): + return 'none' + + if hwaccel_type == 'nvenc': + return 'nvenc' if is_nvidia_runtime_available() else 'none' + + if hwaccel_type == 'vaapi': + return 'vaapi' if is_vaapi_runtime_available() else 'none' + + # Leave explicit QSV/AMF unchanged for non-container users; container auto-detect no longer picks them blindly. + return hwaccel_type + + def get_hwaccel_encoder(hwaccel_type: str) -> str: """ Get the appropriate hardware-accelerated encoder for the given acceleration type. diff --git a/only-vod-chat.py b/only-vod-chat.py index b3a948d..4d9a92d 100644 --- a/only-vod-chat.py +++ b/only-vod-chat.py @@ -228,7 +228,11 @@ class TwitchArchive: elif self.os == 'linux': subprocess.call([bin_path+"/TwitchDownloaderCLI", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"]) if self.username == 'KalathrasLolweapon': print('Uploading html chat to b2 bucket') - subprocess.call(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() except Exception as e: print('A ERROR has ocurred and chat will need to be updated to html manually') @@ -236,9 +240,22 @@ class TwitchArchive: print('Uploading files:') if self.os == 'windows': if self.username == 'KalathrasLolweapon': - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress']) - else:subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + else: + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() elif self.os == 'linux':subprocess.call([bin_path+'/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username]) if self.deleteFiles == 1: diff --git a/run_chat_only.py b/run_chat_only.py new file mode 100644 index 0000000..5134f38 --- /dev/null +++ b/run_chat_only.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +""" +Start chat downloader standalone for testing without recording video. + +Usage: + python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose] + +This script uses the project's `ConfigManager` and `FileManager` to create +appropriate directories and then starts the chat downloader in a background +thread. Press Ctrl+C to stop. +""" +import argparse +import time +from datetime import datetime +import os + +from colorama import Fore, Style + +from modules.config import ConfigManager +from modules.file_manager import FileManager +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system +from modules.downloader import ContentDownloader +from modules.stream_monitor import StreamMonitor + + +def main(): + parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing') + parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name') + parser.add_argument('--output', '-o', help='Output JSON path (optional)') + parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture') + parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds') + parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews') + parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)') + parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method') + parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)') + parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback') + args = parser.parse_args() + + cfg = ConfigManager() + config = cfg.load_streamer_config(args.username) + + # Apply overrides from CLI + if args.use_chat_downloader_primary: + config['useChatDownloaderPrimary'] = True + if args.use_chat_downloader_fallback is not None: + config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback) + + # Ensure directories exist (use configured archive root path) + fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config) + fm.initialize_directories() + + # Build default output path if not provided + if args.output: + json_path = args.output + else: + ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss') + json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json") + + # Initialize downloader + os_type = detect_operating_system() + twitch_downloader_path = get_twitch_downloader_executable(os_type) + ffmpeg_path = get_ffmpeg_executable(os_type) + + downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path, + ffmpeg_path=ffmpeg_path, + config=config) + + print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}") + print(f"Output path: {json_path}") + + stop_requested = {'stop': False} + + def shutdown_check(): + return stop_requested['stop'] + + # Prepare stream monitor + stream_monitor = StreamMonitor(args.username) + + # If chat downloads are disabled in config, enter monitoring mode instead + if not downloader.download_live_chat: + print(f"{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config - entering monitoring mode for {args.username}{Style.RESET_ALL}") + try: + while True: + try: + is_live = stream_monitor.is_user_live() + if is_live: + print(f"{Fore.GREEN}✓ {args.username} is live! Exiting monitor. Run the archiver to record video.{Style.RESET_ALL}") + break + else: + print(f"{Fore.CYAN}{args.username} is offline - checking again in 30s...{Style.RESET_ALL}") + except Exception as e: + print(f"{Fore.YELLOW}⚠ Could not check stream status: {e}{Style.RESET_ALL}") + time.sleep(30) + except KeyboardInterrupt: + print('\nKeyboard interrupt received; stopping monitor...') + return + + # If chat download is enabled, but the stream is currently offline, wait until it goes live + try: + try: + if not stream_monitor.is_user_live(): + print(f"{Fore.CYAN}{args.username} is currently offline - waiting for live stream to start...{Style.RESET_ALL}") + while not stream_monitor.is_user_live(): + time.sleep(10) + except Exception: + # If we cannot determine live status, proceed to start chat downloader anyway + pass + + except KeyboardInterrupt: + print('\nKeyboard interrupt received; exiting...') + return + + # Start thread (stream_monitor passed so downloader can stop when stream ends) + thread = downloader.start_chat_downloader_thread( + args.username, + json_path, + shutdown_check=shutdown_check, + stream_monitor=stream_monitor, + verbose=args.verbose + ) + + try: + if args.foreground: + # Run download directly in foreground + print('Running in foreground; this will block until download completes or interrupted') + success = downloader.download_live_chat_with_chat_downloader( + args.username, + json_path, + max_messages=args.max_messages, + timeout=args.timeout, + shutdown_check=shutdown_check, + stream_monitor=None, + verbose=args.verbose + ) + print('Done, success=' + str(success)) + else: + # Wait for thread to finish or until interrupted + while thread.is_alive(): + time.sleep(0.5) + except KeyboardInterrupt: + print('\nKeyboard interrupt received; stopping downloader...') + stop_requested['stop'] = True + thread.join(timeout=5) + + +if __name__ == '__main__': + main() diff --git a/run_tests.ps1 b/run_tests.ps1 new file mode 100644 index 0000000..f547251 --- /dev/null +++ b/run_tests.ps1 @@ -0,0 +1,37 @@ +# PowerShell script to run Twitch Archive unit tests +# Run this script to execute all unit tests + +Write-Host "======================================================================" -ForegroundColor Cyan +Write-Host "TWITCH ARCHIVE - Running Unit Tests" -ForegroundColor Cyan +Write-Host "======================================================================" -ForegroundColor Cyan +Write-Host "" + +# Check if virtual environment exists and activate it +$venvPath = ".\venv314\Scripts\Activate.ps1" +if (Test-Path $venvPath) { + Write-Host "✓ Activating virtual environment..." -ForegroundColor Green + & $venvPath +} else { + Write-Host "⚠ Virtual environment not found at $venvPath" -ForegroundColor Yellow + Write-Host " Continuing with system Python..." -ForegroundColor Yellow +} + +Write-Host "" + +# Run the tests +Write-Host "Running unit tests..." -ForegroundColor Cyan +python test_twitch_archive_simple.py + +# Check exit code +if ($LASTEXITCODE -eq 0) { + Write-Host "" + Write-Host "======================================================================" -ForegroundColor Green + Write-Host "✓ ALL TESTS PASSED" -ForegroundColor Green + Write-Host "======================================================================" -ForegroundColor Green +} else { + Write-Host "" + Write-Host "======================================================================" -ForegroundColor Red + Write-Host "✗ SOME TESTS FAILED" -ForegroundColor Red + Write-Host "======================================================================" -ForegroundColor Red + exit $LASTEXITCODE +} diff --git a/start_chat_only.bat b/start_chat_only.bat new file mode 100644 index 0000000..e1c4d4a --- /dev/null +++ b/start_chat_only.bat @@ -0,0 +1,18 @@ +rem @echo off +setlocal + +rem Set the path to your virtual environment +set VENV_PATH=.\venv314 + +rem Activate the virtual environment +call "%VENV_PATH%\Scripts\activate.bat" +rem Ensure required packages are installed +pip install -r requirements.txt + +rem Run the standalone chat downloader script +rem Usage: start_chat_only.bat [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary] +rem Pass username as -u and forward additional arguments (mirrors start.bat behavior) +python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9 + +rem Deactivate the virtual environment +call "%VENV_PATH%\Scripts\deactivate.bat" diff --git a/test_twitch_archive.py b/test_twitch_archive.py new file mode 100644 index 0000000..4432d81 --- /dev/null +++ b/test_twitch_archive.py @@ -0,0 +1,592 @@ +""" +Unit tests for Twitch Archive command-line options and configuration. + +Tests focus on: +- Command-line argument parsing (via getopt simulation) +- Options and option combinations +- Configuration loading and merging +- Mode selection logic + +Excludes actual download/processing functionality. + +To run these tests: + python test_twitch_archive.py + or + python -m pytest test_twitch_archive.py -v +""" + +import unittest +import sys +import os +import json +import tempfile +import shutil +import getopt +from unittest.mock import patch, MagicMock, Mock +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from modules.config import ConfigManager +from modules.constants import DEFAULT_CONFIG + + +class TestCommandLineArgumentParsing(unittest.TestCase): + """Test command-line argument parsing logic using getopt directly.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = tempfile.mkdtemp() + self.original_cwd = os.getcwd() + os.chdir(self.test_dir) + + def tearDown(self): + """Clean up test fixtures.""" + os.chdir(self.original_cwd) + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_help_short_option(self): + """Test -h option parsing.""" + argv = ['-h'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + # Should parse successfully + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '-h') + + def test_help_long_option(self): + """Test --help option parsing.""" + argv = ['--help'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '--help') + + def test_username_short_option(self): + """Test -u username option parsing.""" + argv = ['-u', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-u', 'teststreamer')) + + def test_username_long_option(self): + """Test --username option parsing.""" + argv = ['--username', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--username', 'teststreamer')) + + def test_verbose_option(self): + """Test --verbose option parsing.""" + argv = ['--verbose'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--verbose', '')) + + def test_chat_only_option(self): + """Test --chat-only option parsing.""" + argv = ['--chat-only'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--chat-only', '')) + + def test_legacy_option(self): + """Test --legacy option parsing.""" + argv = ['--legacy'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--legacy', '')) + + def test_chat_downloader_options(self): + """Test chat downloader option parsing.""" + argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) + self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) + + def test_legacy_quality_option(self): + """Test -q quality option parsing.""" + argv = ['-q', '720p'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-q', '720p')) + + def test_legacy_boolean_options(self): + """Test legacy boolean option parsing.""" + argv = ['-v', '1', '-c', '0', '-m', '1'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 3) + self.assertEqual(opts[0], ('-v', '1')) + self.assertEqual(opts[1], ('-c', '0')) + self.assertEqual(opts[2], ('-m', '1')) + + def test_invalid_option(self): + """Test that invalid option raises error.""" + argv = ['--invalid-option'] + + with self.assertRaises(getopt.GetoptError): + getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + def test_option_combination_username_verbose(self): + """Test combining -u and --verbose options.""" + argv = ['-u', 'testuser', '--verbose'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('-u', 'testuser')) + self.assertEqual(opts[1], ('--verbose', '')) + + def test_option_combination_all_test_flags(self): + """Test combining all test-related flags.""" + argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 4) + opt_dict = dict(opts) + self.assertEqual(opt_dict['-u'], 'testuser') + self.assertIn('--verbose', opt_dict) + self.assertIn('--chat-only', opt_dict) + self.assertIn('--use-chat-downloader-primary', opt_dict) + + def test_option_combination_legacy_mode_with_overrides(self): + """Test legacy mode with multiple overrides.""" + argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] + opts, args = getopt.getopt( + argv, + "h:u:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 5) + opt_dict = dict(opts) + self.assertIn('--legacy', opt_dict) + self.assertEqual(opt_dict['-q'], '720p') + self.assertEqual(opt_dict['-v'], '1') + self.assertEqual(opt_dict['-c'], '1') + self.assertEqual(opt_dict['-m'], '0') + + +class TestOptionLogicProcessing(unittest.TestCase): + """Test the logic that processes parsed options.""" + + def test_boolean_conversion_true(self): + """Test converting '1' to boolean True.""" + value = '1' + result = bool(int(value)) + self.assertTrue(result) + + def test_boolean_conversion_false(self): + """Test converting '0' to boolean False.""" + value = '0' + result = bool(int(value)) + self.assertFalse(result) + + def test_chat_only_auto_enables_verbose(self): + """Test that chat-only mode should auto-enable verbose.""" + # Simulate the logic from main() + chat_only_mode = True + verbose_mode = False + + if chat_only_mode: + verbose_mode = True + + self.assertTrue(verbose_mode) + + def test_default_chat_downloader_fallback(self): + """Test that chat downloader fallback defaults to enabled.""" + use_chat_downloader_fallback = True # Default + + # Unless explicitly disabled + self.assertTrue(use_chat_downloader_fallback) + + def test_mode_selection_legacy_with_config_json(self): + """Test mode selection logic when config.json exists.""" + # Simulate conditions + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = False + + # Logic from main() + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + def test_mode_selection_multi_streamer_with_global_json(self): + """Test mode selection logic when global.json exists.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_multi_streamer_with_username_flag(self): + """Test mode selection when -u flag is used.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = 'testuser' + global_config_exists = False + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_explicit_legacy_flag(self): + """Test mode selection with explicit --legacy flag.""" + use_legacy_mode = True + legacy_config_exists = False + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + +class TestConfigManager(unittest.TestCase): + """Test ConfigManager functionality.""" + + def setUp(self): + """Set up test fixtures.""" + self.test_dir = tempfile.mkdtemp() + self.original_cwd = os.getcwd() + os.chdir(self.test_dir) + + # Create config directory structure + os.makedirs('config/streamers', exist_ok=True) + + # Patch ConfigManager to use test directory + self.config_dir_patch = patch.object( + ConfigManager, + '__init__', + lambda self: self._init_with_test_dir(self.test_dir) + ) + + def _init_with_test_dir(self, test_dir): + """Initialize ConfigManager with test directory.""" + self.config_dir = Path(test_dir) / "config" + self.streamers_dir = self.config_dir / "streamers" + self.global_config = self._load_global_config() + + def tearDown(self): + """Clean up test fixtures.""" + os.chdir(self.original_cwd) + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_load_global_config_default(self): + """Test loading default configuration when global.json doesn't exist.""" + # Patch the config_dir to use temp directory + with patch.object(ConfigManager, '_ConfigManager__init__') as mock_init: + manager = ConfigManager.__new__(ConfigManager) + manager.config_dir = Path(self.test_dir) / "config" + manager.streamers_dir = manager.config_dir / "streamers" + manager.global_config = manager._load_global_config() + + # Should have default config values + self.assertIsNotNone(manager.global_config) + self.assertIn('username', manager.global_config) + self.assertIn('quality', manager.global_config) + + def test_load_global_config_from_file(self): + """Test loading global configuration from file.""" + # Create global config + global_config = { + 'quality': '720p', + 'downloadVOD': False, + 'downloadCHAT': True + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # Should merge with defaults + self.assertEqual(manager.global_config['quality'], '720p') + self.assertFalse(manager.global_config['downloadVOD']) + self.assertTrue(manager.global_config['downloadCHAT']) + + def test_load_global_config_filters_comments(self): + """Test that global config filters out comment fields.""" + global_config = { + '_comment': 'This is a comment', + 'quality': '720p', + '_note': 'Another comment' + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # Comments should be filtered out + self.assertNotIn('_comment', manager.global_config) + self.assertNotIn('_note', manager.global_config) + self.assertEqual(manager.global_config['quality'], '720p') + + def test_load_global_config_filters_schema(self): + """Test that global config filters out $schema field.""" + global_config = { + '$schema': './global.schema.json', + 'quality': '720p' + } + + with open('config/global.json', 'w') as f: + json.dump(global_config, f) + + manager = ConfigManager() + + # $schema should be filtered out + self.assertNotIn('$schema', manager.global_config) + self.assertEqual(manager.global_config['quality'], '720p') + + def test_load_streamer_config_new_streamer(self): + """Test loading config for a new streamer (doesn't exist yet).""" + manager = ConfigManager() + + config = manager.load_streamer_config('newstreamer') + + # Should create default config + self.assertEqual(config['username'], 'newstreamer') + self.assertTrue(config['enabled']) + + # Config file should be created + self.assertTrue(os.path.exists('config/streamers/newstreamer.json')) + + def test_load_streamer_config_existing_streamer(self): + """Test loading config for existing streamer.""" + # Create streamer config + streamer_config = { + 'username': 'existingstreamer', + 'enabled': True, + 'quality': 'source', + 'downloadVOD': True + } + + with open('config/streamers/existingstreamer.json', 'w') as f: + json.dump(streamer_config, f) + + manager = ConfigManager() + config = manager.load_streamer_config('existingstreamer') + + # Should load streamer config + self.assertEqual(config['username'], 'existingstreamer') + self.assertEqual(config['quality'], 'source') + self.assertTrue(config['downloadVOD']) + + def test_load_streamer_config_merges_with_global(self): + """Test that streamer config merges with global config.""" + # Create global config + with open('config/global.json', 'w') as f: + json.dump({ + 'quality': '720p', + 'downloadVOD': True, + 'downloadCHAT': False + }, f) + + # Create streamer config with override + with open('config/streamers/teststreamer.json', 'w') as f: + json.dump({ + 'username': 'teststreamer', + 'enabled': True, + 'quality': 'source' # Override global + }, f) + + manager = ConfigManager() + config = manager.load_streamer_config('teststreamer') + + # Should have streamer's quality (override) + self.assertEqual(config['quality'], 'source') + # Should have global's downloadVOD (inherited) + self.assertTrue(config['downloadVOD']) + # Should have global's downloadCHAT (inherited) + self.assertFalse(config['downloadCHAT']) + + def test_load_streamer_config_filters_comments(self): + """Test that streamer config filters out comments.""" + with open('config/streamers/teststreamer.json', 'w') as f: + json.dump({ + '_comment': 'Test comment', + 'username': 'teststreamer', + 'enabled': True, + '_note': 'Another note' + }, f) + + manager = ConfigManager() + config = manager.load_streamer_config('teststreamer') + + # Comments should be filtered + self.assertNotIn('_comment', config) + self.assertNotIn('_note', config) + self.assertEqual(config['username'], 'teststreamer') + + def test_get_all_enabled_streamers_empty(self): + """Test getting enabled streamers when none exist.""" + manager = ConfigManager() + + streamers = manager.get_all_enabled_streamers() + + self.assertEqual(len(streamers), 0) + + def test_get_all_enabled_streamers_with_enabled(self): + """Test getting enabled streamers.""" + # Create multiple streamer configs + with open('config/streamers/streamer1.json', 'w') as f: + json.dump({'username': 'streamer1', 'enabled': True}, f) + + with open('config/streamers/streamer2.json', 'w') as f: + json.dump({'username': 'streamer2', 'enabled': True}, f) + + with open('config/streamers/streamer3.json', 'w') as f: + json.dump({'username': 'streamer3', 'enabled': False}, f) + + manager = ConfigManager() + streamers = manager.get_all_enabled_streamers() + + # Should return only enabled streamers + self.assertEqual(len(streamers), 2) + self.assertIn('streamer1', streamers) + self.assertIn('streamer2', streamers) + self.assertNotIn('streamer3', streamers) + + def test_get_all_enabled_streamers_default_enabled(self): + """Test that streamers are enabled by default if not specified.""" + # Create config without explicit enabled field + with open('config/streamers/streamer1.json', 'w') as f: + json.dump({'username': 'streamer1'}, f) + + manager = ConfigManager() + streamers = manager.get_all_enabled_streamers() + + # Should be included (defaults to enabled) + self.assertIn('streamer1', streamers) + + def test_create_default_streamer_config(self): + """Test creating default streamer config.""" + manager = ConfigManager() + manager.create_default_streamer_config('newstreamer') + + # File should exist + config_file = Path('config/streamers/newstreamer.json') + self.assertTrue(config_file.exists()) + + # Should have correct structure + with open(config_file) as f: + config = json.load(f) + + self.assertEqual(config['username'], 'newstreamer') + self.assertTrue(config['enabled']) + self.assertIn('$schema', config) + + + + + +if __name__ == '__main__': + # Run tests with verbose output + print("="*70) + print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") + print("="*70) + print() + unittest.main(verbosity=2) diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py new file mode 100644 index 0000000..445e741 --- /dev/null +++ b/test_twitch_archive_simple.py @@ -0,0 +1,762 @@ +""" +Unit tests for Twitch Archive command-line options and configuration. + +Tests focus on: +- Command-line argument parsing (via getopt simulation) +- Options and option combinations +- Configuration logic (filtering, merging, etc.) +- Mode selection logic + +Excludes actual download/processing functionality. + +To run these tests: + python test_twitch_archive_simple.py + or + python -m pytest test_twitch_archive_simple.py -v +""" + +import unittest +import sys +import os +import json +import getopt +import tempfile +import importlib.util +from pathlib import Path +from unittest.mock import patch, MagicMock, Mock + +# Add parent directory to path for imports +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from modules.constants import DEFAULT_CONFIG +from modules.file_manager import FileManager +from modules.downloader import ContentDownloader +from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_hardware_acceleration, resolve_hwaccel_type + + +def load_twitch_archive_module(): + """Load the main script module for targeted regression tests.""" + module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py') + spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestCommandLineArgumentParsing(unittest.TestCase): + """Test command-line argument parsing logic using getopt directly.""" + + def test_help_short_option(self): + """Test -h option parsing.""" + argv = ['-h'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + # Should parse successfully + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '-h') + + def test_help_long_option(self): + """Test --help option parsing.""" + argv = ['--help'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0][0], '--help') + + def test_username_short_option(self): + """Test -u username option parsing.""" + argv = ['-u', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-u', 'teststreamer')) + + def test_username_long_option(self): + """Test --username option parsing.""" + argv = ['--username', 'teststreamer'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--username', 'teststreamer')) + + def test_verbose_option(self): + """Test --verbose option parsing.""" + argv = ['--verbose'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--verbose', '')) + + def test_chat_only_option(self): + """Test --chat-only option parsing.""" + argv = ['--chat-only'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--chat-only', '')) + + def test_rclone_smoke_test_option(self): + """Test --rclone-smoke-test option parsing.""" + argv = ['--rclone-smoke-test'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--rclone-smoke-test', '')) + + def test_healthcheck_option(self): + """Test --healthcheck option parsing.""" + argv = ['--healthcheck'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--healthcheck', '')) + + def test_legacy_option(self): + """Test --legacy option parsing.""" + argv = ['--legacy'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('--legacy', '')) + + def test_chat_downloader_options(self): + """Test chat downloader option parsing.""" + argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) + self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) + + def test_legacy_quality_option(self): + """Test -q quality option parsing.""" + argv = ['-q', '720p'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 1) + self.assertEqual(opts[0], ('-q', '720p')) + + def test_legacy_boolean_options(self): + """Test legacy boolean option parsing.""" + argv = ['-v', '1', '-c', '0', '-m', '1'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 3) + self.assertEqual(opts[0], ('-v', '1')) + self.assertEqual(opts[1], ('-c', '0')) + self.assertEqual(opts[2], ('-m', '1')) + + def test_invalid_option(self): + """Test that invalid option raises error.""" + argv = ['--invalid-option'] + + with self.assertRaises(getopt.GetoptError): + getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + def test_option_combination_username_verbose(self): + """Test combining -u and --verbose options.""" + argv = ['-u', 'testuser', '--verbose'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 2) + self.assertEqual(opts[0], ('-u', 'testuser')) + self.assertEqual(opts[1], ('--verbose', '')) + + def test_option_combination_all_test_flags(self): + """Test combining all test-related flags.""" + argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 4) + opt_dict = dict(opts) + self.assertEqual(opt_dict['-u'], 'testuser') + self.assertIn('--verbose', opt_dict) + self.assertIn('--chat-only', opt_dict) + self.assertIn('--use-chat-downloader-primary', opt_dict) + + def test_option_combination_legacy_mode_with_overrides(self): + """Test legacy mode with multiple overrides.""" + argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] + opts, args = getopt.getopt( + argv, + "hu:q:a:v:c:m:r:d:n:", + ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", + "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + ) + + self.assertEqual(len(opts), 5) + opt_dict = dict(opts) + self.assertIn('--legacy', opt_dict) + self.assertEqual(opt_dict['-q'], '720p') + self.assertEqual(opt_dict['-v'], '1') + self.assertEqual(opt_dict['-c'], '1') + self.assertEqual(opt_dict['-m'], '0') + + +class TestOptionLogicProcessing(unittest.TestCase): + """Test the logic that processes parsed options.""" + + def test_boolean_conversion_true(self): + """Test converting '1' to boolean True.""" + value = '1' + result = bool(int(value)) + self.assertTrue(result) + + def test_boolean_conversion_false(self): + """Test converting '0' to boolean False.""" + value = '0' + result = bool(int(value)) + self.assertFalse(result) + + def test_chat_only_auto_enables_verbose(self): + """Test that chat-only mode should auto-enable verbose.""" + # Simulate the logic from main() + chat_only_mode = True + verbose_mode = False + + if chat_only_mode: + verbose_mode = True + + self.assertTrue(verbose_mode) + + def test_default_chat_downloader_fallback(self): + """Test that chat downloader fallback defaults to enabled.""" + use_chat_downloader_fallback = True # Default value + + # Unless explicitly disabled + self.assertTrue(use_chat_downloader_fallback) + + def test_mode_selection_legacy_with_config_json(self): + """Test mode selection logic when config.json exists.""" + # Simulate conditions + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = False + + # Logic from main() function + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + def test_mode_selection_multi_streamer_with_global_json(self): + """Test mode selection logic when global.json exists.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_multi_streamer_with_username_flag(self): + """Test mode selection when -u flag is used.""" + use_legacy_mode = False + legacy_config_exists = True + specific_streamer = 'testuser' + global_config_exists = False + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertFalse(should_use_legacy) + + def test_mode_selection_explicit_legacy_flag(self): + """Test mode selection with explicit --legacy flag.""" + use_legacy_mode = True + legacy_config_exists = False + specific_streamer = None + global_config_exists = True + + should_use_legacy = use_legacy_mode or ( + legacy_config_exists and not specific_streamer and not global_config_exists + ) + + self.assertTrue(should_use_legacy) + + +class TestConfigLogic(unittest.TestCase): + """Test configuration management logic.""" + + def test_comment_filtering_logic(self): + """Test that comment fields are filtered out.""" + user_config = { + '_comment': 'This is a comment', + 'quality': '720p', + '_note': 'Another comment', + 'username': 'testuser' + } + + # Apply the filtering logic (same as in ConfigManager) + filtered = {k: v for k, v in user_config.items() if not k.startswith('_')} + + self.assertNotIn('_comment', filtered) + self.assertNotIn('_note', filtered) + self.assertIn('quality', filtered) + self.assertIn('username', filtered) + self.assertEqual(filtered['quality'], '720p') + + def test_schema_filtering_logic(self): + """Test that $schema field is filtered out.""" + user_config = { + '$schema': './config.schema.json', + 'quality': '720p', + 'username': 'testuser' + } + + # Apply the filtering logic (same as in ConfigManager) + filtered = {k: v for k, v in user_config.items() + if not k.startswith('_') and k != '$schema'} + + self.assertNotIn('$schema', filtered) + self.assertIn('quality', filtered) + self.assertIn('username', filtered) + + def test_config_merging_logic(self): + """Test config merging logic (streamer overrides global).""" + global_config = { + 'quality': '720p', + 'downloadVOD': True, + 'downloadCHAT': False, + 'username': 'default' + } + + streamer_config = { + 'username': 'specificstreamer', + 'quality': 'source', # Override + # downloadVOD and downloadCHAT inherited from global + } + + # Simulate merging (same as in ConfigManager.load_streamer_config) + merged = global_config.copy() + merged.update(streamer_config) + + # Check overrides + self.assertEqual(merged['quality'], 'source') # Overridden + self.assertEqual(merged['username'], 'specificstreamer') # Overridden + # Check inherited values + self.assertTrue(merged['downloadVOD']) + self.assertFalse(merged['downloadCHAT']) + + def test_default_config_structure(self): + """Test that DEFAULT_CONFIG has expected keys.""" + self.assertIn('username', DEFAULT_CONFIG) + self.assertIn('quality', DEFAULT_CONFIG) + self.assertIn('downloadVOD', DEFAULT_CONFIG) + self.assertIn('downloadCHAT', DEFAULT_CONFIG) + self.assertIn('downloadMETADATA', DEFAULT_CONFIG) + self.assertIn('uploadCloud', DEFAULT_CONFIG) + self.assertIn('deleteFiles', DEFAULT_CONFIG) + self.assertIn('notifications', DEFAULT_CONFIG) + self.assertIn('refresh', DEFAULT_CONFIG) + + def test_enabled_flag_filtering_logic(self): + """Test filtering streamers by enabled flag.""" + streamers = [ + {'username': 'streamer1', 'enabled': True}, + {'username': 'streamer2', 'enabled': True}, + {'username': 'streamer3', 'enabled': False}, + {'username': 'streamer4'}, # No enabled field + ] + + # Simulate filtering logic (same as in ConfigManager.get_all_enabled_streamers) + enabled_streamers = [ + s['username'] for s in streamers + if s.get('enabled', True) # Default to True if not specified + ] + + self.assertIn('streamer1', enabled_streamers) + self.assertIn('streamer2', enabled_streamers) + self.assertNotIn('streamer3', enabled_streamers) + self.assertIn('streamer4', enabled_streamers) # Default enabled + + def test_default_streamer_config_structure(self): + """Test default streamer config structure.""" + # Simulate what create_default_streamer_config creates + username = 'newstreamer' + default_config = { + "$schema": "../streamer.schema.json", + "username": username, + "enabled": True + } + + self.assertEqual(default_config['username'], username) + self.assertTrue(default_config['enabled']) + self.assertIn('$schema', default_config) + + +class TestFileManagerUploadPaths(unittest.TestCase): + """Test rclone upload path preparation.""" + + def test_build_upload_relative_paths_uses_forward_slashes(self): + """Rclone --files-from entries must use POSIX separators on Windows.""" + with tempfile.TemporaryDirectory() as temp_dir: + manager = FileManager( + root_path=temp_dir, + username='testuser', + config={ + 'uploadCloud': True, + 'uploadPreMergeVideo': True, + 'uploadMergedVideo': True, + 'uploadChatVideo': True + } + ) + + relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s') + + self.assertTrue(relative_paths) + self.assertTrue(all('\\' not in path for path in relative_paths)) + self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths) + self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths) + + +class TestDownloaderConfiguration(unittest.TestCase): + """Regression tests for downloader config wiring.""" + + def test_download_vod_method_not_shadowed_by_boolean_flag(self): + """Config booleans must not overwrite callable downloader methods.""" + downloader = ContentDownloader( + twitch_downloader_path='TwitchDownloaderCLI', + ffmpeg_path='ffmpeg', + config={ + 'downloadVOD': True, + 'downloadCHAT': True, + 'downloadLiveCHAT': True + } + ) + + self.assertTrue(callable(downloader.download_vod)) + self.assertTrue(downloader.download_vod_enabled) + + +class TestLinuxToolResolution(unittest.TestCase): + """Ensure Linux containers prefer their own installed toolchain.""" + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_ffmpeg(self, mock_which): + mock_which.return_value = '/usr/bin/ffmpeg' + + self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg') + + @patch('modules.utils.shutil.which') + def test_linux_prefers_system_twitch_downloader(self, mock_which): + mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI' + + self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') + + @patch('modules.utils.os.path.exists', return_value=False) + @patch('modules.utils.shutil.which', return_value=None) + def test_linux_auto_hwaccel_falls_back_to_software_without_runtime(self, _mock_which, _mock_exists): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'none') + self.assertEqual(resolve_hwaccel_type(detected, 'linux'), 'none') + + @patch('modules.utils.shutil.which', return_value='/usr/bin/nvidia-smi') + def test_linux_auto_hwaccel_uses_nvenc_when_nvidia_runtime_visible(self, _mock_which): + detected = detect_hardware_acceleration('auto', 'linux') + + self.assertEqual(detected, 'nvenc') + + +class TestChatRenderBehavior(unittest.TestCase): + """Regression tests for chat rendering defaults and retry behavior.""" + + @patch('modules.downloader.sys.platform', 'linux') + @patch('modules.downloader.subprocess.Popen') + def test_linux_chat_render_uses_container_safe_font(self, mock_popen): + with tempfile.TemporaryDirectory() as temp_dir: + json_path = os.path.join(temp_dir, 'chat.json') + video_path = os.path.join(temp_dir, 'chat.mp4') + + with open(json_path, 'w', encoding='utf-8') as handle: + json.dump( + { + 'comments': [ + { + 'message': {'body': 'hello world ' * 20}, + 'commenter': {'display_name': 'tester'} + } + ] + }, + handle + ) + + class FakeProcess: + def __init__(self, output_path: str): + self.stdout = iter(['rendering']) + self.output_path = output_path + + def wait(self): + with open(self.output_path, 'wb') as handle: + handle.write(b'x' * 2048) + return 0 + + captured = {} + + def build_process(cmd, **_kwargs): + captured['cmd'] = cmd + return FakeProcess(video_path) + + mock_popen.side_effect = build_process + + downloader = ContentDownloader('TwitchDownloaderCLI', 'ffmpeg', {'downloadCHAT': True}) + + result = downloader.render_chat(json_path, video_path, '-c:v libx264 "{save_path}"') + + self.assertTrue(result) + self.assertIn('DejaVu Sans', captured['cmd']) + + +class TestMultiStreamerCleanupRegression(unittest.TestCase): + """Regression tests for multi-streamer conversion and cleanup behavior.""" + + def setUp(self): + self.module = load_twitch_archive_module() + + def _build_archiver(self, temp_dir: str, upload_cloud: bool = True): + archiver = MagicMock() + archiver.username = 'maddoscientist0' + archiver.os_type = 'linux' + archiver.quality = 'best' + archiver.downloadLiveCHAT = False + archiver.downloadCHAT = False + archiver.downloadVOD = False + archiver.downloadMETADATA = False + archiver.mergeVideoChat = False + archiver.mergeChatLayout = 'side-by-side' + archiver.onlyRaw = False + archiver.vodTimeout = 0 + archiver.shutdown_requested = False + archiver.deleteFiles = True + archiver.uploadCloud = upload_cloud + + archiver.notification_manager = MagicMock() + archiver.recorder = MagicMock() + archiver.processor = MagicMock() + archiver.downloader = MagicMock() + archiver.stream_monitor = MagicMock() + archiver.file_manager = MagicMock() + archiver.file_manager.raw_path = Path(temp_dir) / 'raw' + archiver.file_manager.video_path = Path(temp_dir) / 'video' + archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json' + archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat' + + os.makedirs(archiver.file_manager.raw_path, exist_ok=True) + os.makedirs(archiver.file_manager.video_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True) + os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True) + + return archiver + + def test_process_stream_keeps_raw_when_conversion_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.processor.process_raw_stream.return_value = False + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_called_once() + + def test_process_stream_only_deletes_rendered_files_after_real_upload(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir, upload_cloud=False) + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + archiver.recorder.record.side_effect = write_raw_file + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_called_once() + archiver.file_manager.delete_local_files.assert_not_called() + + upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] + self.assertFalse(upload_filename_base.startswith('LIVE_')) + + def test_process_stream_preserves_files_when_chat_render_fails(self): + manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') + + with tempfile.TemporaryDirectory() as temp_dir: + archiver = self._build_archiver(temp_dir) + archiver.downloadCHAT = True + archiver.downloadVOD = True + archiver.vodTimeout = 30 + archiver.processor.process_raw_stream.return_value = True + archiver.file_manager.upload_to_cloud.return_value = True + archiver.downloader.last_chat_render_attempted = False + archiver.downloader.last_chat_render_succeeded = False + + def write_raw_file(_stream_info, raw_path): + with open(raw_path, 'wb') as handle: + handle.write(b'x' * 4096) + return True + + def failed_chat_render(*_args, **_kwargs): + archiver.downloader.last_chat_render_attempted = True + archiver.downloader.last_chat_render_succeeded = False + return False + + archiver.recorder.record.side_effect = write_raw_file + archiver.downloader.download_and_render_chat.side_effect = failed_chat_render + archiver.stream_monitor.get_latest_vod.return_value = { + 'data': { + 'user': { + 'videos': { + 'edges': [ + { + 'node': { + 'id': '2756589076', + 'title': 'Test', + 'recordedAt': '2026-04-25T09:14:01Z' + } + } + ] + } + } + } + } + + stream_info = { + 'title': 'Test', + 'createdAt': '2026-04-25T09:14:01Z' + } + + manager._process_stream(archiver, stream_info, 'stream-id') + + archiver.file_manager.clean_raw_file.assert_not_called() + archiver.file_manager.delete_local_files.assert_not_called() + + +if __name__ == '__main__': + # Run tests with verbose output + print("="*70) + print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") + print("="*70) + print() + unittest.main(verbosity=2) diff --git a/twitch-archive.py b/twitch-archive.py index 5844cde..707bdfe 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -34,6 +34,8 @@ import time import json import signal import getopt +import pathlib +import subprocess from typing import Dict, Optional, Any from datetime import datetime, timedelta @@ -48,7 +50,8 @@ from modules.config import ConfigManager from modules.notifications import NotificationManager from modules.utils import ( detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader + get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader, + verify_rclone, get_env_value ) from modules.stream_monitor import StreamMonitor from modules.recorder import StreamRecorder @@ -147,10 +150,25 @@ class TwitchArchive: Raises: SystemExit: If .env file is not found """ - if not load_dotenv(find_dotenv()): + dotenv_loaded = load_dotenv(find_dotenv()) + has_required_env = bool( + get_env_value('CLIENT-ID', 'CLIENT_ID') and + get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') + ) + + if not dotenv_loaded and has_required_env: + print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') + return + + if not dotenv_loaded and not has_required_env: print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') + sys.exit(1) + + if not has_required_env: + print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') sys.exit(1) def _initialize_components(self) -> None: @@ -259,6 +277,8 @@ class TwitchArchive: verify_ffmpeg(self.os_type) if self.downloadVOD or self.downloadCHAT: verify_twitch_downloader(self.os_type) + if self.uploadCloud and not verify_rclone(): + sys.exit(1) # Print configuration summary self._print_configuration_summary() @@ -404,7 +424,8 @@ class TwitchArchive: print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') # Process the raw stream file - self.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) + self.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -426,7 +447,11 @@ class TwitchArchive: else: # Get video duration first (needed for chat conversion and trimming) ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + video_duration = get_video_duration(live_proc_path, ffmpeg_path) print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') # Convert chat format if needed (chat_downloader uses different JSON structure) @@ -561,7 +586,17 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - self.file_manager.clean_raw_file(live_raw_path) + chat_render_retry_needed = ( + self.downloader.last_chat_render_attempted and + not self.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: + self.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = self.file_manager.upload_to_cloud( @@ -570,7 +605,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and upload_success: + if self.deleteFiles and self.uploadCloud and upload_success and not chat_render_retry_needed: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -890,6 +925,8 @@ class TwitchArchiveManager: verify_ffmpeg(first_archiver.os_type) if first_archiver.downloadVOD or first_archiver.downloadCHAT: verify_twitch_downloader(first_archiver.os_type) + if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone(): + sys.exit(1) # Print configuration summary for each streamer for username, archiver in self.archivers.items(): @@ -1018,7 +1055,7 @@ class TwitchArchiveManager: # Generate timestamp and filename timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" + filename_base = f"{archiver.username}_{timestamp}" # Parse stream start time live_date = datetime.strptime( @@ -1029,8 +1066,8 @@ class TwitchArchiveManager: raw_extension = '.ts' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") + live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}") chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification @@ -1088,6 +1125,7 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, verbose=self.verbose ) except Exception as e: @@ -1116,6 +1154,7 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, + stream_monitor=archiver.stream_monitor, verbose=self.verbose or self.chat_only ) # Wait for completion @@ -1170,8 +1209,10 @@ class TwitchArchiveManager: print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') # Process raw stream + processing_succeeded = False if not archiver.onlyRaw: - archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) + archiver.downloader.reset_chat_render_status() # Wait for live chat download if it was started live_chat_downloaded = False @@ -1210,8 +1251,12 @@ class TwitchArchiveManager: chat_rendered_successfully = False else: # Get video duration first - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + if not processing_succeeded or not os.path.exists(live_proc_path): + print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') + video_duration = None + else: + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) if video_duration is None: print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') @@ -1360,7 +1405,17 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - archiver.file_manager.clean_raw_file(live_raw_path) + chat_render_retry_needed = ( + archiver.downloader.last_chat_render_attempted and + not archiver.downloader.last_chat_render_succeeded + ) + + if chat_render_retry_needed: + print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') + elif processing_succeeded: + archiver.file_manager.clean_raw_file(live_raw_path) + elif os.path.exists(live_raw_path): + print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') # Upload to cloud if configured upload_success = archiver.file_manager.upload_to_cloud( @@ -1369,7 +1424,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and upload_success: + if archiver.deleteFiles and archiver.uploadCloud and upload_success and not chat_render_retry_needed: archiver.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1384,6 +1439,79 @@ class TwitchArchiveManager: ) +def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int: + """Run a one-off rclone smoke test using the configured upload destination.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + if not enabled_streamers: + print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}') + print(f'{Fore.CYAN}→ Use -u or enable a streamer config{Style.RESET_ALL}') + return 1 + username = enabled_streamers[0] + + config = config_manager.load_streamer_config(username) + file_manager = FileManager( + root_path=config.get('root_path', 'archive'), + username=username, + config=config + ) + file_manager.initialize_directories() + + print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}') + print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}') + print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "")}{Style.RESET_ALL}\n') + + return 0 if file_manager.run_rclone_smoke_test() else 1 + + +def run_healthcheck(specific_streamer: Optional[str] = None) -> int: + """Run a local readiness check suitable for Docker health checks.""" + config_manager = ConfigManager() + + if specific_streamer: + username = specific_streamer + else: + enabled_streamers = config_manager.get_all_enabled_streamers() + username = enabled_streamers[0] if enabled_streamers else 'vinesauce' + + config = config_manager.load_streamer_config(username) + archive = TwitchArchive(config) + + try: + archive._load_environment_variables() + except SystemExit: + return 1 + + archive._initialize_components() + + checks_ok = True + if not verify_streamlink(): + checks_ok = False + if not verify_ffmpeg(archive.os_type): + checks_ok = False + if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type): + checks_ok = False + if archive.uploadCloud: + if not verify_rclone(): + checks_ok = False + rclone_config_path = os.getenv('RCLONE_CONFIG') + if rclone_config_path and not os.path.exists(rclone_config_path): + print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}') + checks_ok = False + + if not checks_ok: + return 1 + + print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}') + return 0 + + # ============================================================================ # COMMAND-LINE INTERFACE # ============================================================================ @@ -1399,6 +1527,8 @@ def main(argv: list) -> None: """ specific_streamer = None use_legacy_mode = False + rclone_smoke_test_mode = False + healthcheck_mode = False help_msg = f''' {Fore.CYAN}{"=" * 70} @@ -1425,6 +1555,8 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving --legacy Force legacy mode (use config.json) --chat-only Test mode: Only download chat (skip video recording) Automatically enables verbose logging + --healthcheck Validate config and tool availability, then exit + --rclone-smoke-test Create a small test file and upload it with rclone --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) --no-chat-downloader-fallback Disable chat_downloader fallback @@ -1462,7 +1594,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving "h:u:q:a:v:c:m:r:d:n:", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] ) except getopt.GetoptError as e: print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') @@ -1489,6 +1621,10 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving elif opt == "--chat-only": chat_only_mode = True verbose_mode = True # Auto-enable verbose for chat-only mode + elif opt == "--healthcheck": + healthcheck_mode = True + elif opt == "--rclone-smoke-test": + rclone_smoke_test_mode = True elif opt == "--legacy": use_legacy_mode = True elif opt == "--use-chat-downloader-primary": @@ -1521,6 +1657,12 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving legacy_overrides['deleteFiles'] = bool(int(arg)) elif opt in ("-n", "--notifications"): legacy_overrides['notifications'] = bool(int(arg)) + + if rclone_smoke_test_mode: + sys.exit(run_rclone_smoke_test(specific_streamer)) + + if healthcheck_mode: + sys.exit(run_healthcheck(specific_streamer)) # Determine which mode to use if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):