diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 49463af..0000000 --- a/.dockerignore +++ /dev/null @@ -1,18 +0,0 @@ -.git -.github -.forgejo -.vs -.vscode -venv*/ -__pycache__/ -*.pyc -archive/ -dotnet/ -tests/ -.pytest_cache/ -bin/temp/ -bin/ffmpeg -bin/ffmpeg.exe -bin/ffprobe -bin/TwitchDownloaderCLI -bin/TwitchDownloaderCLI.exe \ No newline at end of file diff --git a/.env.development b/.env.development deleted file mode 100644 index 4ccc47f..0000000 --- a/.env.development +++ /dev/null @@ -1,16 +0,0 @@ -TWITCH_ARCHIVE_DEV_IMAGE=twitch-archive-local -TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive-dev -TWITCH_ARCHIVE_APP_ENV_FILE=./.env.development -TWITCH_ARCHIVE_ARCHIVE_BIND=./archive -TWITCH_ARCHIVE_CONFIG_BIND=./config -TWITCH_ARCHIVE_ARGS=-u vinesauce --verbose -TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce -TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf -PYTHONUNBUFFERED=1 -TZ=UTC -CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua -CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 -OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm -SENDER= -RECEIVER= -PASSWD= \ No newline at end of file diff --git a/.env.production b/.env.production deleted file mode 100644 index b1c4197..0000000 --- a/.env.production +++ /dev/null @@ -1,16 +0,0 @@ -TWITCH_ARCHIVE_IMAGE=forgejo.maddoscientisto.net/maddo/twitch-archive:latest -TWITCH_ARCHIVE_CONTAINER_NAME=twitch-archive -TWITCH_ARCHIVE_APP_ENV_FILE=./.env.production -TWITCH_ARCHIVE_ARCHIVE_BIND=./archive -TWITCH_ARCHIVE_CONFIG_BIND=./config -TWITCH_ARCHIVE_ARGS=-u vinesauce -TWITCH_ARCHIVE_HEALTHCHECK_STREAMER=vinesauce -TWITCH_ARCHIVE_RCLONE_CONFIG=/app/config/rclone.conf -PYTHONUNBUFFERED=1 -TZ=UTC -CLIENT-ID=vdyevjvllziylzwsm3y925p79pwtua -CLIENT-SECRET=y906xadsmf22q54suuzsmfnfav3jc7 -OAUTH-PRIVATE-TOKEN=ll4kvlmxuajfgi9lgi5d8mkeglsyvm -SENDER= -RECEIVER= -PASSWD= \ No newline at end of file diff --git a/.forgejo/workflows/publish-python-container.yml b/.forgejo/workflows/publish-python-container.yml deleted file mode 100644 index 831fa51..0000000 --- a/.forgejo/workflows/publish-python-container.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: Publish Twitch Archive Container - -on: - push: - branches: - - master - - main - paths: - - docker/python.Dockerfile - - docker/entrypoint.sh - - docker-compose.yml - - docker-compose.override.yml - - requirements.txt - - twitch-archive.py - - run_chat_only.py - - modules/** - - .forgejo/workflows/publish-python-container.yml - workflow_dispatch: - -env: - REGISTRY: ${{ vars.FORGEJO_REGISTRY }} - IMAGE_NAMESPACE: ${{ vars.IMAGE_NAMESPACE }} - IMAGE_NAME: ${{ vars.IMAGE_NAME != '' && vars.IMAGE_NAME || 'twitch-archive' }} - BUILD_CONTEXT: . - DOCKERFILE_PATH: docker/python.Dockerfile - -jobs: - publish: - runs-on: docker - env: - DOCKER_HOST: ${{ vars.DOCKER_HOST != '' && vars.DOCKER_HOST || 'tcp://172.17.0.1:2375' }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Validate workflow variables - run: | - set -eu - if [ -z "${REGISTRY}" ]; then echo "vars.FORGEJO_REGISTRY is required"; exit 1; fi - if [ -z "${IMAGE_NAMESPACE}" ]; then echo "vars.IMAGE_NAMESPACE is required"; exit 1; fi - if [ ! -f "${DOCKERFILE_PATH}" ]; then echo "Dockerfile not found at ${DOCKERFILE_PATH}"; exit 1; fi - if [ ! -f "requirements.txt" ]; then echo "requirements.txt is missing"; exit 1; fi - - - name: Ensure Docker CLI exists - run: | - set -eu - if command -v docker >/dev/null 2>&1; then - docker --version - exit 0 - fi - - ARCH="$(uname -m)" - case "${ARCH}" in - x86_64) DOCKER_ARCH="x86_64" ;; - aarch64|arm64) DOCKER_ARCH="aarch64" ;; - *) echo "Unsupported architecture for Docker CLI bootstrap: ${ARCH}"; exit 1 ;; - esac - - DOCKER_CLI_VERSION="27.5.1" - curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" -o docker.tgz - tar -xzf docker.tgz - mkdir -p "${HOME}/.local/bin" - mv docker/docker "${HOME}/.local/bin/docker" - chmod +x "${HOME}/.local/bin/docker" - echo "${HOME}/.local/bin" >> "${FORGEJO_PATH}" - "${HOME}/.local/bin/docker" --version - - - name: Ensure Docker Buildx exists - run: | - set -eu - if docker buildx version >/dev/null 2>&1; then - docker buildx version - exit 0 - fi - - ARCH="$(uname -m)" - case "${ARCH}" in - x86_64) BUILDX_ARCH="amd64" ;; - aarch64|arm64) BUILDX_ARCH="arm64" ;; - *) echo "Unsupported architecture for Docker Buildx bootstrap: ${ARCH}"; exit 1 ;; - esac - - BUILDX_VERSION="v0.21.1" - mkdir -p "${HOME}/.docker/cli-plugins" - curl -fsSL "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-${BUILDX_ARCH}" -o "${HOME}/.docker/cli-plugins/docker-buildx" - chmod +x "${HOME}/.docker/cli-plugins/docker-buildx" - docker buildx version - - - name: Check Docker daemon connectivity - run: | - set -eu - echo "Using DOCKER_HOST=${DOCKER_HOST}" - docker version - docker info >/dev/null - - - name: Create Buildx builder - run: | - set -eu - docker buildx rm forgejo-builder >/dev/null 2>&1 || true - docker buildx create --name forgejo-builder --driver docker-container --use - docker buildx inspect --bootstrap - - - name: Validate registry secrets - run: | - set -eu - if [ -z "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" ]; then echo "secrets.FORGEJO_REGISTRY_USERNAME is required"; exit 1; fi - if [ -z "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" ]; then echo "secrets.FORGEJO_REGISTRY_TOKEN is required"; exit 1; fi - - - name: Login to registry - run: | - set -eu - echo "${{ secrets.FORGEJO_REGISTRY_TOKEN }}" | docker login "${REGISTRY}" -u "${{ secrets.FORGEJO_REGISTRY_USERNAME }}" --password-stdin - - - name: Build and push image - run: | - set -eu - IMAGE_REF="${REGISTRY}/${IMAGE_NAMESPACE}/${IMAGE_NAME}" - SHA_TAG="${IMAGE_REF}:sha-${FORGEJO_SHA}" - BRANCH_TAG="${IMAGE_REF}:${FORGEJO_REF_NAME}" - docker buildx build \ - --file "${DOCKERFILE_PATH}" \ - --tag "${SHA_TAG}" \ - --tag "${BRANCH_TAG}" \ - --tag "${IMAGE_REF}:latest" \ - --push \ - "${BUILD_CONTEXT}" \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index f887d97..8e459b5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,22 +1,6 @@ -# Git LFS tracking for large media and binaries - -# Binaries in bin/ -bin/* filter=lfs diff=lfs merge=lfs -text -bin/TwitchDownloaderCLI* filter=lfs diff=lfs merge=lfs -text -bin/ffmpeg filter=lfs diff=lfs merge=lfs -text -# Video files under archive (raw/ and common video extensions) -archive/**/video/** filter=lfs diff=lfs merge=lfs -text -archive/**/video/raw/** filter=lfs diff=lfs merge=lfs -text -*.mp4 filter=lfs diff=lfs merge=lfs -text -*.mkv filter=lfs diff=lfs merge=lfs -text -*.mov filter=lfs diff=lfs merge=lfs -text -# Large archives and executables -*.zip filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.exe filter=lfs diff=lfs merge=lfs -text -*.dll filter=lfs diff=lfs merge=lfs -text # Auto detect text files and perform LF normalization * text=auto +bin/ffmpeg filter=lfs diff=lfs merge=lfs -text bin/ffmpeg.exe filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI filter=lfs diff=lfs merge=lfs -text bin/TwitchDownloaderCLI.exe filter=lfs diff=lfs merge=lfs -text diff --git a/.github/instructions/blazor.instructions.md b/.github/instructions/blazor.instructions.md deleted file mode 100644 index 4e88cc0..0000000 --- a/.github/instructions/blazor.instructions.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -description: 'Blazor component and application patterns' -applyTo: '**/*.razor, **/*.razor.cs, **/*.razor.css' ---- - -## Blazor Code Style and Structure - -- Write idiomatic and efficient Blazor and C# code. -- Follow .NET and Blazor conventions. -- Use Razor Components appropriately for component-based UI development. -- Prefer inline functions for smaller components but separate complex logic into code-behind or service classes. -- Async/await should be used where applicable to ensure non-blocking UI operations. - -## Naming Conventions - -- Follow PascalCase for component names, method names, and public members. -- Use camelCase for private fields and local variables. -- Prefix interface names with "I" (e.g., IUserService). - -## Blazor and .NET Specific Guidelines - -- Utilize Blazor's built-in features for component lifecycle (e.g., OnInitializedAsync, OnParametersSetAsync). -- Use data binding effectively with @bind. -- Leverage Dependency Injection for services in Blazor. -- Structure Blazor components and services following Separation of Concerns. -- Always use the latest version C#, currently C# 13 features like record types, pattern matching, and global usings. - -## Error Handling and Validation - -- Implement proper error handling for Blazor pages and API calls. -- Use logging for error tracking in the backend and consider capturing UI-level errors in Blazor with tools like ErrorBoundary. -- Implement validation using FluentValidation or DataAnnotations in forms. - -## Blazor API and Performance Optimization - -- Utilize Blazor server-side or WebAssembly optimally based on the project requirements. -- Use asynchronous methods (async/await) for API calls or UI actions that could block the main thread. -- Optimize Razor components by reducing unnecessary renders and using StateHasChanged() efficiently. -- Minimize the component render tree by avoiding re-renders unless necessary, using ShouldRender() where appropriate. -- Use EventCallbacks for handling user interactions efficiently, passing only minimal data when triggering events. - -## Caching Strategies - -- Implement in-memory caching for frequently used data, especially for Blazor Server apps. Use IMemoryCache for lightweight caching solutions. -- For Blazor WebAssembly, utilize localStorage or sessionStorage to cache application state between user sessions. -- Consider Distributed Cache strategies (like Redis or SQL Server Cache) for larger applications that need shared state across multiple users or clients. -- Cache API calls by storing responses to avoid redundant calls when data is unlikely to change, thus improving the user experience. - -## State Management Libraries - -- Use Blazor's built-in Cascading Parameters and EventCallbacks for basic state sharing across components. -- Implement advanced state management solutions using libraries like Fluxor or BlazorState when the application grows in complexity. -- For client-side state persistence in Blazor WebAssembly, consider using Blazored.LocalStorage or Blazored.SessionStorage to maintain state between page reloads. -- For server-side Blazor, use Scoped Services and the StateContainer pattern to manage state within user sessions while minimizing re-renders. - -## API Design and Integration - -- Use HttpClient or other appropriate services to communicate with external APIs or your own backend. -- Implement error handling for API calls using try-catch and provide proper user feedback in the UI. - -## Testing and Debugging in Visual Studio - -- All unit testing and integration testing should be done in Visual Studio. -- Test Blazor components and services using xUnit, NUnit, or MSTest. -- Use Moq or NSubstitute for mocking dependencies during tests. -- Debug Blazor UI issues using browser developer tools and Visual Studio's debugging tools for backend and server-side issues. -- For performance profiling and optimization, rely on Visual Studio's diagnostics tools. - -## Security and Authentication - -- Implement Authentication and Authorization in the Blazor app where necessary using ASP.NET Identity or JWT tokens for API authentication. -- Use HTTPS for all web communication and ensure proper CORS policies are implemented. - -## API Documentation and Swagger - -- Use Swagger/OpenAPI for API documentation for your backend API services. -- Ensure XML documentation for models and API methods for enhancing Swagger documentation. diff --git a/.gitignore b/.gitignore index 49c4387..b1a3beb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,6 @@ config/global.json # Streamer-specific configurations (personal settings) config/streamers/*.json -config/rclone.conf -config/rclone.conf.* # Python cache __pycache__/ @@ -23,49 +21,4 @@ venv3/** .gitignore bin/** \n+# Ignore any virtual environment directories starting with 'venv' (venv, venv3, venv314, etc.) -venv*/ -.vs/ProjectEvaluation/twitch-archive-2.metadata.v10.bin -.vs/ProjectEvaluation/twitch-archive-2.projects.v10.bin -.vs/ProjectEvaluation/twitch-archive-2.strings.v10.bin -.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/CodeChunks.db -.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/SemanticSymbols.db -.vs/Twitch-Archive-2/DesignTimeBuild/.dtbcache.v2 -.vs/Twitch-Archive-2/FileContentIndex/843065c8-d80f-4907-b0ae-6d010b3a5699.vsidx -.vs/Twitch-Archive-2/FileContentIndex/ef7e1a3c-80cd-4867-a9a8-2e5099471227.vsidx -.vs/Twitch-Archive-2/v18/.futdcache.v2 -.vs/Twitch-Archive-2/v18/.suo -.vs/Twitch-Archive-2/v18/DocumentLayout.backup.json -.vs/Twitch-Archive-2/v18/DocumentLayout.json -.vscode/settings.json - -# C# / Visual Studio -# Build Folders -bin/ -obj/ - -# Visual Studio files -*.user -*.suo -*.userprefs -*.csproj.user -*.pidb -*.pdb -*.cache -*.ilk -*.log -*.vspscc -*.vssscc - -# Test results and packages -TestResults/ -packages/ -*.nupkg - -# Database and backup -*.dbmdl -*.bak -*.backup -*.orig - -dotnet/.vs/** -.vs/** \ No newline at end of file +venv*/ \ No newline at end of file diff --git a/README.md b/README.md index 979e223..40d7471 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,8 @@ # Twitch Archive Inspired by https://github.com/EnterGin/Auto-Stream-Recording-Twitch -## Git LFS - -This repository stores large media files (recorded video and some binaries). Use Git LFS to manage large objects. - -Quick setup (Windows): - -1. Install Git LFS: `git lfs install` -2. Ensure `.gitattributes` is committed (this repo includes one). -3. If you already have large files tracked by normal Git, migrate them: - -```powershell -git lfs install -git lfs track "*.mp4" "*.mkv" "bin/*" -git add .gitattributes -git add -A -git commit -m "Migrate large files to LFS" -git push origin main -``` - -Notes: -- Git LFS needs server-side support. If using GitHub, enable Git LFS on the remote and ensure you have sufficient bandwidth/storage quota. -- You can customize tracked patterns in `.gitattributes`. - Python script to check, download live stream, VOD, chat and upload them to any cloud storage supported by rclone. -## Docker - -This repository now includes a Python-only container setup for the archiver. The dotnet subapp is not part of this container flow. - -Files: - -- `docker/python.Dockerfile`: production image for the Python archiver -- `docker-compose.yml`: deployment-oriented compose file -- `docker-compose.override.yml`: local development and testing override -- `.env.production`: production container and app environment template -- `.env.development`: development container and app environment template -- `dockerstart.bat`: Windows helper to run the container like the old batch launcher - -### Container layout - -- Mount your external archive folder to `/app/archive` -- Mount your external config folder to `/app/config` -- Put your `rclone.conf` file at `/app/config/rclone.conf` on the mounted host path -- The container exports `RCLONE_CONFIG=/app/config/rclone.conf`, so rclone will use that file automatically - -### Production deployment - -1. Edit `.env.production` with your image name, Twitch credentials, bind paths, and default arguments. -2. Place your streamer JSON files and `rclone.conf` in the mounted config folder. -3. Start the container: - -```powershell -docker compose --env-file .env.production up -d -``` - -4. Follow logs: - -```powershell -docker compose --env-file .env.production logs -f twitch-archive -``` - -### Development and local testing - -The override compose file builds the image locally and mounts the repository for faster iteration. - -Start it with: - -```powershell -docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml up --build -``` - -Run a one-off manual test for another streamer: - -```powershell -docker compose --env-file .env.development -f docker-compose.yml -f docker-compose.override.yml run --rm twitch-archive python twitch-archive.py -u hackerling --verbose -``` - -Or use the Windows helper: - -```powershell -.\dockerstart.bat vinesauce --verbose -``` - -That batch launcher mirrors the old pattern and expands to a compose `run` command, so you can test any streamer manually. - -If the host has the NVIDIA Container Toolkit installed and you want FFmpeg/NVENC inside the container, use the optional NVIDIA override: - -```powershell -.\dockerrebuild.bat -.\dockerstart.bat --nvidia vinesauce --verbose -``` - -The image built by `.\dockerrebuild.bat` already includes the NVIDIA-capable FFmpeg/container toolchain. The optional [docker-compose.nvidia.yml](docker-compose.nvidia.yml) layer is only for runtime GPU passthrough: it requests `gpus: all` and sets `NVIDIA_VISIBLE_DEVICES` plus `NVIDIA_DRIVER_CAPABILITIES=compute,utility,video` for the container. - -On systems without NVIDIA support, keep using the normal command without `--nvidia`; the image still builds the same way, it just runs without GPU passthrough. - -### Healthcheck and smoke tests - -- Container healthcheck command: `python twitch-archive.py --healthcheck -u vinesauce` -- Rclone smoke test command: `python twitch-archive.py -u vinesauce --rclone-smoke-test` - -The healthcheck verifies config loading plus `streamlink`, `ffmpeg`, `TwitchDownloaderCLI`, and `rclone` availability. The smoke test writes a tiny file, uploads it with the configured rclone remote, and prints the live rclone output into the container logs. - ## ⚡ FFmpeg 8.0 Enhanced Now with FFmpeg 8.0+ support featuring hardware acceleration and performance improvements! - **5-10x faster encoding** with NVIDIA, Intel, or AMD GPUs diff --git a/UpgradePlan.md b/UpgradePlan.md deleted file mode 100644 index 6654d3d..0000000 --- a/UpgradePlan.md +++ /dev/null @@ -1,214 +0,0 @@ -Plan: C# .NET 10 Twitch Archive Rewrite -A complete port of the Python archiver to C# .NET 10 with Blazor Server UI, real-time process output, SQLite state tracking, full DI/service pattern, NLog logging, and a resilient recording engine. Placed under dotnet/ in the existing repo. - -Project Layout -Step 1 — Solution & Project scaffolding -Create the SLN and three projects: - -TwitchArchive.Core — classlib, targets net10.0 -TwitchArchive.Web — Blazor Server (blazorserver), targets net10.0 -TwitchArchive.Tests — xUnit, targets net10.0 -NuGet packages: - -Core: Microsoft.EntityFrameworkCore.Sqlite, Polly, NLog, NLog.Extensions.Logging -Web: all Core packages + Microsoft.AspNetCore.SignalR, NLog.Web.AspNetCore -Tests: xunit, Moq, coverlet.collector, Microsoft.EntityFrameworkCore.InMemory -Step 2 — Configuration models -Mirror the existing JSON schemas as C# POCOs with System.Text.Json attributes: - -GlobalConfig.cs — one property per key in config/global.json.example -StreamerConfig.cs — all fields nullable (override semantics), only Username and Enabled required -EffectiveConfig.cs — computed merge of global + per-streamer; exposes resolved values -AppSettings.cs — app-level settings (password hash, tool paths, .env secrets) -IConfigurationService / ConfigurationService: - -LoadGlobal() / SaveGlobal(GlobalConfig) -LoadStreamer(string username) / SaveStreamer(StreamerConfig) -GetAllStreamers(), GetEffectiveConfig(string username) (merge logic) -Reads/writes global.json and config/streamers/*.json — same files as Python -Step 3 — Infrastructure layer -TwitchApiClient (injectable, mockable): - -GetOAuthTokenAsync() — POST to https://id.twitch.tv/oauth2/token, caches token, refreshes on 401 -CheckStreamStatusAsync(string username) — GQL query for live stream + archiveVideo.id -GetLatestVodAsync(string username) — GQL query for most recent VOD -ValidateUsernameAsync(string username) — Helix /users endpoint -Credentials read from environment (CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN) -All methods return typed result objects, never throw on network errors — return Result (or OneOf) -HttpResiliencePolicy (Polly): - -Wraps HttpClient for TwitchApiClient -WaitAndRetryForever with exponential backoff starting at 15 s, doubling, capped at 10 minutes -Only applies to transient errors (5xx, timeout, HttpRequestException) — not 401/404 -Logged via NLog on each retry attempt -ProcessRunner (injectable + mockable for tests): - -RunAsync(ProcessRunOptions options, CancellationToken ct) → int exitCode -StartAsync(ProcessRunOptions options, CancellationToken ct) → IRunningProcess handle (for long-lived processes like streamlink) -Reads stdout and stderr line by line asynchronously -Reports each line to IProcessOutputStore (streamer + job context) -Forwards to NLog -ProcessRunOptions: FileName, Arguments, WorkingDirectory, RedirectOutput - -Step 4 — Core services (all behind interfaces) -IStreamMonitorService / StreamMonitorService - -Wraps TwitchApiClient -CheckIsLiveAsync(string username) → LiveStreamInfo? -GetLatestVodAsync(string username) → VodInfo? -IRecorderService / RecorderService - -StartRecordingAsync(string username, string quality, string outputPath, CancellationToken ct) → Task -Invokes streamlink via ProcessRunner -Passes --hls-live-restart, --stream-segment-threads, optional OAuth header -Returns when streamlink exits (either stream ended or ct cancelled) -IProcessorService / ProcessorService - -ProcessRawStreamAsync(string rawPath, string outputPath, EffectiveConfig cfg, CancellationToken ct) -Builds ffmpeg args: hwaccel, thread count, error recovery flags, faststart, copy codecs -MergeVideoChatAsync(string videoPath, string chatVideoPath, string outputPath, string layout, CancellationToken ct) -IDownloaderService / DownloaderService - -DownloadVodAsync(VodInfo vod, string outputPath, EffectiveConfig cfg, CancellationToken ct) → bool -Invokes TwitchDownloaderCLI videodownload -Chat download methods stubbed with NotImplementedException / commented structure; interface is defined now to keep the architecture clean -IUploadService / UploadService - -UploadAsync(string localRoot, IEnumerable relativeFilePaths, string rcloneDest, CancellationToken ct) → bool -Writes a temp files-from list, invokes rclone copy --files-from -Returns success/failure; preserves local files on failure -IFileManagerService / FileManagerService - -InitializeDirectories(string rootPath, string username) -GetPaths(string rootPath, string username, string filenameBase) → ArchivePaths record (all expected paths) -CleanRawFile(string path, bool cleanRaw) -DeleteLocalFiles(ArchivePaths paths, EffectiveConfig cfg) -GetUniquePath(string path) → adds numeric suffix if file exists -Step 5 — Recording resilience engine -RecoveryPolicy (POCO, unit-testable, no DI deps): - -Encodes a state machine with these states: - -State Meaning -Monitoring Normal polling at refresh interval -Recording streamlink subprocess active -FastReconnect Stream ended; checking every 10 s for up to 2 minutes -SlowReconnect Still not live after 2 min; checking every 60 s concurrently with post-processing -PostProcessing Confirmed ended; ffmpeg / VOD download / upload running -NetworkFault Twitch API unreachable; exponential back-off (30 s → capped at 10 min) -Transitions: - -Recording → streamlink exits → enter FastReconnect, record phase start time -FastReconnect → live confirmed → start new Recording (new filename/segment) -FastReconnect (2 min elapsed) → enter SlowReconnect + kick off PostProcessing concurrently -SlowReconnect → live confirmed → start new Recording -SlowReconnect / Monitoring → API call throws network error → enter NetworkFault -NetworkFault → successful API response → return to previous state (Monitoring or re-enter FastReconnect if we were mid-reconnect) -NetworkFault backoff: 30s * 2^attempt, capped at 600s -RecoveryPolicy is a pure class with a Tick(DateTime now, bool? isLive, bool networkError) method → returns RecoveryDecision (what to do next + sleep duration). Fully unit-testable with no async or DI. - -StreamWorker : BackgroundService - -One instance per enabled streamer -Holds RecoveryPolicy instance -Main loop: evaluate policy decision → execute the corresponding service call → loop -Started/stopped by StreamWorkerManager -Writes job records to SQLite on start/complete/fail -StreamWorkerManager - -StartWorker(string username), StopWorker(string username), RestartWorker(string username) -Called at app startup for all enabled streamers -Called from Web UI on enable/disable/config change -Workers stored in ConcurrentDictionary -Step 6 — Persistence (SQLite + EF Core) -ArchiveDbContext with three tables: - -StreamSessions: Id, StreamerUsername, TwitchStreamId, Title, StartedAt, EndedAt, Status (Recording/Processing/Uploading/Complete/Failed) - -ArchiveJobs: Id, SessionId, JobType (enum: RecordLive, ProcessLive, DownloadVod, ProcessVod, UploadCloud, DeleteLocal), Status, StartedAt, CompletedAt, FilePath, ErrorMessage - -StreamerStates: Username, IsMonitoring, LastCheckedAt, CurrentRecoveryState - -Migrations via EF Core CLI. ISessionRepository / IJobRepository interfaces for testability with in-memory EF provider in tests. - -Step 7 — Process output streaming -IProcessOutputStore: - -AppendLine(string streamerId, Guid jobId, string line, bool isError) -GetRecentLines(string streamerId, Guid jobId, int count = 500) → IReadOnlyList -In-memory circular buffer (1000 lines per job, last 20 jobs per streamer) -ProcessOutputHub : Hub (SignalR): - -Clients call SubscribeToStreamer(string username) → join group streamer:{username} -Clients call SubscribeToJob(Guid jobId) → join group job:{jobId} -Server pushes ReceiveLine(OutputLine line) from ProcessRunner via IHubContext -On subscribe: server immediately sends buffered lines from IProcessOutputStore -Step 8 — Blazor Server Web UI -Authentication: Cookie-based single-password auth via ASP.NET Core minimal auth middleware. Password stored as BCrypt hash in AppSettings. Login.razor page at /login. Protected routes with [Authorize]. - -Pages & Components: - -Dashboard.razor (/) — grid of all configured streamers showing: username, live/offline badge, current recovery state, last recorded session, quick Start/Stop monitoring toggle - -StreamerDetail.razor (/streamer/{username}) — live status, current job pipeline steps (record → process → upload with progress), ProcessOutputConsole.razor showing real-time terminal output via SignalR - -ProcessOutputConsole.razor — reusable Blazor component; subscribes to SignalR on mount, renders an auto-scrolling
 with colored output (stdout = white, stderr = orange/red), handles reconnect
-
-Sessions.razor (/sessions) — paginated list of past archive sessions with job statuses and expandable per-job output
-
-GlobalConfig.razor (/config/global) — EditForm bound to GlobalConfig model with data annotations validation, Save button calls IConfigurationService.SaveGlobal()
-
-StreamerConfig.razor (/config/{username}) — similar form for per-streamer overrides; each field has a nullable toggle (inherit from global vs override)
-
-AddStreamer.razor (/config/new) — minimal form: username + enabled; creates new config/streamers/{username}.json
-
-AppSettings.razor (/settings) — tool paths (streamlink, ffmpeg, TwitchDownloaderCLI, rclone), change password
-
-Step 9 — NLog configuration
-nlog.config (XML): two targets:
-
-Console (colored, with level formatting)
-File rolling (logs/archive-${shortdate}.log, keep 30 days)
-Log structured context: StreamerUsername, JobId, JobType as NLog ScopeContext properties. Service methods open a scope via ILogger.BeginScope(...).
-
-Step 10 — Docker
-Dockerfile (multi-stage):
-
-Build stage: mcr.microsoft.com/dotnet/sdk:10.0
-Runtime stage: mcr.microsoft.com/dotnet/aspnet:10.0 (Linux)
-Install ffmpeg, streamlink (via pip), download TwitchDownloaderCLI binary for linux-x64
-rclone installed via shell script or apt
-Expose port 8080
-ENTRYPOINT ["dotnet", "TwitchArchive.Web.dll"]
-docker-compose.yml:
-
-Volume mounts: ./config:/app/config, ./archive:/app/archive, ./logs:/app/logs
-Environment variables: CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN
-Windows dev: run directly with dotnet run; tool paths auto-detected (Windows vs Linux) via RuntimeInformation.IsOSPlatform(OSPlatform.Windows) in ToolPathResolver
-
-Step 11 — Unit tests
-TwitchArchive.Tests covers:
-
-RecoveryPolicyTests — state machine transitions, timing phases, network fault backoff; pure synchronous tests
-ConfigurationServiceTests — JSON load/save/merge with temp files
-TwitchApiClientTests — mocked HttpMessageHandler; OAuth, GQL queries, 401 refresh, network errors
-FileManagerServiceTests — path generation, directory creation with temp directories
-DownloaderServiceTests / RecorderServiceTests — mocked ProcessRunner; verify correct CLI arguments
-UploadServiceTests — mocked ProcessRunner; verify rclone argument construction
-SessionRepositoryTests — EF Core in-memory provider
-EffectiveConfigTests — global + streamer override merge logic
-Verification
-dotnet build dotnet/TwitchArchive.sln — zero warnings, zero errors
-dotnet test dotnet/TwitchArchive.Tests/ — all tests green
-docker compose up --build in the dotnet/ folder → app reachable at http://localhost:8080
-Manual: add a test streamer config, enable monitoring, confirm it polls and records a live stream, runs ffmpeg, uploads via rclone
-Resilience manual test: kill network during recording → verify FastReconnect phase kicks in, resumes after connectivity restored
-Decisions
-
-Blazor Server chosen for real-time terminal output without a separate API; no WASM needed
-streamlink for live + TwitchDownloaderCLI for VOD (same split as Python; streamlink gives better live resilience)
-Simple BCrypt password auth (not full Identity — this is a single-user tool)
-RecoveryPolicy as a pure POCO state machine keeps the resilience logic fully unit-testable without async/mocking
-Polly WaitAndRetryForever on HttpClient handles persistent network failure independently of the application-level state machine; they are complementary — Polly handles individual HTTP call retries, RecoveryPolicy handles the overall workflow state
-Chat download service interface is defined in Step 4 but methods are stubbed — adding implementation later requires only filling in DownloaderService without touching any other layer
-Config files remain in the same format/location so the Python and C# versions can share the same config directory
\ No newline at end of file
diff --git a/UpgradePlan2.md b/UpgradePlan2.md
deleted file mode 100644
index cc4ffbd..0000000
--- a/UpgradePlan2.md
+++ /dev/null
@@ -1,78 +0,0 @@
-
-
-
@Body
``` Add a top bar (`
`) with the app title "Twitch Archive" and a hamburger toggle ` - - - @if (showAdvanced) - { -
-

Advanced per-streamer defaults

-
- - -
-
- - -
-
- - -
-
- - -
-
- - - - - - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - - - - - - - - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- } - -
- -
- - -@code { - private TwitchArchive.Core.Config.StreamerConfig model = new() { Enabled = true }; - private bool showAdvanced = false; - - // local fields for binding nullable/global override values - private bool downloadVOD = true; - private bool downloadCHAT = true; - private bool downloadLiveCHAT = true; - private bool mergeVideoChat = false; - private int? vodTimeout; - private bool uploadPreMergeVideo = true; - private bool uploadMergedVideo = true; - private bool uploadChatVideo = false; - private bool deleteFiles = false; - private bool onlyRaw = false; - private bool cleanRaw = true; - private int? hlsSegments; - private int? hlsSegmentsVOD; - private bool streamlinkTtvlol = false; - private string? ffmpegHwaccel; - private int? ffmpegThreads; - private string? ffmpegAudioCodec; - private int? ffmpegAudioSamplerate; - private string? ffmpegAudioBitrate; - private bool ffmpegErrorRecovery = true; - private bool ffmpegFaststart = true; - private bool ffmpegProgress = false; - - private void ToggleAdvanced() => showAdvanced = !showAdvanced; - - private void Save() - { - model.Username = model.Username?.Trim().ToLowerInvariant() ?? string.Empty; - if (string.IsNullOrWhiteSpace(model.Username)) return; - - // map local fields into nullable model properties - model.DownloadVOD = downloadVOD; - model.DownloadCHAT = downloadCHAT; - model.DownloadLiveCHAT = downloadLiveCHAT; - model.MergeVideoChat = mergeVideoChat; - model.VodTimeout = vodTimeout; - model.UploadPreMergeVideo = uploadPreMergeVideo; - model.UploadMergedVideo = uploadMergedVideo; - model.UploadChatVideo = uploadChatVideo; - model.DeleteFiles = deleteFiles; - model.OnlyRaw = onlyRaw; - model.CleanRaw = cleanRaw; - model.HlsSegments = hlsSegments; - model.HlsSegmentsVOD = hlsSegmentsVOD; - model.StreamlinkTtvlol = streamlinkTtvlol; - model.FfmpegHwaccel = ffmpegHwaccel; - model.FfmpegThreads = ffmpegThreads; - model.FfmpegAudioCodec = ffmpegAudioCodec; - model.FfmpegAudioSamplerate = ffmpegAudioSamplerate; - model.FfmpegAudioBitrate = ffmpegAudioBitrate; - model.FfmpegErrorRecovery = ffmpegErrorRecovery; - model.FfmpegFaststart = ffmpegFaststart; - model.FfmpegProgress = ffmpegProgress; - - ConfigService.SaveStreamer(model); - Nav.NavigateTo($"/config/{model.Username}"); - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor deleted file mode 100644 index a666c0b..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor +++ /dev/null @@ -1,230 +0,0 @@ -@page "/settings" -@using System.Text.Json -@inject TwitchArchive.Core.Config.IConfigurationService ConfigService -@inject TwitchArchive.Web.Services.IAuthService Auth - -

Settings

- -@if (saved) -{ -
Saved.
-} - - -
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- -
- - @if (showDefaults) - { - - -
-

Defaults

-
- - -
-
- - -
-
- - -
-
- - -
-
- - - - - - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - - - - - - - - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- -
-
-
- } -
- -
- -
- -

Change Password

-@if (!string.IsNullOrEmpty(pwError)) -{ -
@pwError
-} -
- - - - -
- -@code { - private TwitchArchive.Core.Config.GlobalConfig globalModel = new(); - private bool saved = false; - private bool showDefaults = false; - private string currentPw = string.Empty; - private string newPw = string.Empty; - private string confirmPw = string.Empty; - private string pwError = string.Empty; - - protected override void OnInitialized() - { - Load(); - } - - private void Load() - { - try - { - globalModel = ConfigService.LoadGlobal() ?? new TwitchArchive.Core.Config.GlobalConfig(); - } - catch { globalModel = new TwitchArchive.Core.Config.GlobalConfig(); } - } - - private void SaveGlobal() - { - try - { - ConfigService.SaveGlobal(globalModel); - saved = true; - // optionally notify auth or other services - } - catch { } - } - - private void ToggleDefaults() => showDefaults = !showDefaults; - - private void ChangePassword() - { - pwError = string.Empty; - if (!Auth.ValidatePassword(currentPw)) { pwError = "Current password incorrect"; return; } - if (string.IsNullOrWhiteSpace(newPw)) { pwError = "New password required"; return; } - if (newPw != confirmPw) { pwError = "Passwords do not match"; return; } - var hash = BCrypt.Net.BCrypt.HashPassword(newPw); - Auth.SetPasswordHash(hash); - Auth.Refresh(); - saved = true; - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor deleted file mode 100644 index 1ec6d41..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor +++ /dev/null @@ -1,10 +0,0 @@ -@page "/config/global" -@inject NavigationManager Nav - -@code { - protected override void OnInitialized() - { - // Consolidated settings are now at /settings - Nav.NavigateTo("/settings", true); - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Index.razor b/dotnet/src/TwitchArchive.Web/Pages/Index.razor deleted file mode 100644 index e25b8cd..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/Index.razor +++ /dev/null @@ -1,96 +0,0 @@ -@page "/" -@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager -@inject TwitchArchive.Web.Services.SessionCacheService SessionCache - -

Dashboard

- -@if (streamers.Count == 0) -{ -
No streamers configured. Add one on the Add Streamer page.
-} - -
- @foreach (var s in streamers) - { -
-
- @s - Edit - @(WorkerManager.IsRunning(s) ? "Live" : "Offline") -
-
-
Last session: @(lastStarts.ContainsKey(s) ? lastStarts[s].ToLocalTime().ToString() : "-")
-
- - -
-
-
- } -
- -@* Show global feedback when there are streamers but no recent sessions *@ -@if (streamers.Count > 0 && (lastStarts == null || lastStarts.Count == 0)) -{ -
No recent sessions found for configured streamers.
-} - -@code { - private List streamers = new(); - private Dictionary lastStarts = new(); - - private void OnCacheUpdatedHandler() - { - _ = InvokeAsync(() => { - lastStarts = SessionCache.GetSnapshot(); - StateHasChanged(); - }); - } - - protected override async Task OnInitializedAsync() - { - LoadStreamers(); - lastStarts = SessionCache.GetSnapshot(); - SessionCache.Updated += OnCacheUpdatedHandler; - } - - private void LoadStreamers() - { - // Try to find the config/streamers folder from the app content root and parent folders. - string? cfgDir = FindConfigStreamersFolder(); - if (!string.IsNullOrEmpty(cfgDir) && Directory.Exists(cfgDir)) - { - streamers = Directory.GetFiles(cfgDir, "*.json").Select(f => Path.GetFileNameWithoutExtension(f)).ToList(); - } - } - - private string? FindConfigStreamersFolder() - { - // Prefer ContentRoot if available, fall back to Environment.CurrentDirectory. - var start = AppContext.BaseDirectory ?? Environment.CurrentDirectory; - var dir = new DirectoryInfo(start); - for (int i = 0; i < 6 && dir != null; i++) - { - var candidate = Path.Combine(dir.FullName, "config", "streamers"); - if (Directory.Exists(candidate)) return candidate; - dir = dir.Parent; - } - - // final attempt: repo-root relative (use project parent heuristics) - var alt = Path.Combine(Environment.CurrentDirectory, "..", "..", "..", "config", "streamers"); - try { alt = Path.GetFullPath(alt); } catch { } - if (Directory.Exists(alt)) return alt; - return null; - } - - // Index reads from the singleton SessionCacheService; updates are pushed via the Updated event. - - private void Start(string u) { WorkerManager.StartWorker(u); } - private async Task Stop(string u) { await WorkerManager.StopWorkerAsync(u); } - - public async ValueTask DisposeAsync() - { - SessionCache.Updated -= OnCacheUpdatedHandler; - await Task.CompletedTask; - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Login.razor b/dotnet/src/TwitchArchive.Web/Pages/Login.razor deleted file mode 100644 index ff4f1da..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/Login.razor +++ /dev/null @@ -1,20 +0,0 @@ -@page "/login" -@attribute [AllowAnonymous] -@using Microsoft.AspNetCore.Components - -

Login

- -@if (error) -{ -
Invalid password
-} - -
- - -
- -@code { - [Parameter] - public bool error { get; set; } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Media.razor b/dotnet/src/TwitchArchive.Web/Pages/Media.razor deleted file mode 100644 index a42c78c..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/Media.razor +++ /dev/null @@ -1,59 +0,0 @@ -@page "/media" -@inject TwitchArchive.Core.Config.IConfigurationService ConfigService - -

Media Library

- -@if (string.IsNullOrWhiteSpace(archiveRoot)) -{ -
Archive root is not configured. Set it on the Settings page.
-} -else if (!Directory.Exists(archiveRoot)) -{ -
Archive root '@archiveRoot' does not exist on disk.
-} -else -{ - @if (entries.Count == 0) - { -
No media files found in '@archiveRoot'.
- } - else - { - @foreach (var kv in entries) - { -
-

@kv.Key

-
    - @foreach (var f in kv.Value) - { -
  • @f.Name - @((f.Length/1024.0/1024.0).ToString("0.00")) MB - @f.LastWriteTime.ToLocalTime()
  • - } -
-
- } - } -} - -@code { - private string? archiveRoot; - private Dictionary> entries = new(); - - protected override void OnInitialized() - { - var g = ConfigService.LoadGlobal(); - archiveRoot = g?.ArchiveRoot; - if (!string.IsNullOrWhiteSpace(archiveRoot) && Directory.Exists(archiveRoot)) - { - var di = new DirectoryInfo(archiveRoot); - foreach (var dir in di.GetDirectories()) - { - try - { - var files = dir.GetFiles("*.mp4").Concat(dir.GetFiles("*.mkv")).Concat(dir.GetFiles("*.ts")).Concat(dir.GetFiles("*.flv")).OrderByDescending(f => f.LastWriteTime).ToList(); - if (files.Count > 0) entries[dir.Name] = files; - } - catch { } - } - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor deleted file mode 100644 index 4cd79de..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor +++ /dev/null @@ -1,37 +0,0 @@ -@page "/monitor" -@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager -@inject TwitchArchive.Core.Services.IProcessOutputStore OutputStore -@using TwitchArchive.Web.Shared - -

Streamer Monitor

- -
- - - -
- -
- Status: @status -
- - - -@code { - private string username = "hackerling"; - private string status = "idle"; - - private void Start() - { - if (string.IsNullOrWhiteSpace(username)) return; - WorkerManager.StartWorker(username); - status = WorkerManager.IsRunning(username) ? "running" : "starting"; - } - - private async Task Stop() - { - if (string.IsNullOrWhiteSpace(username)) return; - await WorkerManager.StopWorkerAsync(username); - status = WorkerManager.IsRunning(username) ? "running" : "stopped"; - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor deleted file mode 100644 index 256da43..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor +++ /dev/null @@ -1,73 +0,0 @@ -@page "/sessions" -@using TwitchArchive.Core.Persistence.Models -@inject TwitchArchive.Core.Persistence.ISessionRepository SessionRepo - -

Sessions

- - - - - - - - - @foreach (var s in sessions) - { - - - - - - - - - @if (expandedSession == s.Id) - { - - } - } - -
IdStreamerStartedEndedStatusJobs
@s.Id@s.StreamerUsername@s.StartedAt.ToLocalTime()@(s.EndedAt?.ToLocalTime().ToString() ?? "-")@s.Status
-
    - @if (jobs?.Any() ?? false) - { - @foreach (var j in jobs) - { -
  • @j.Id - @j.JobType - @j.Status - @j.StartedAt.ToLocalTime() - @(j.FilePath ?? "")
  • - } - } - else - { -
  • No jobs
  • - } -
-
- -@code { - private List sessions = new(); - private List? jobs; - private long? expandedSession; - - protected override async Task OnInitializedAsync() - { - await Refresh(); - } - - private async Task Refresh() - { - sessions = await SessionRepo.GetRecentSessionsAsync(50); - StateHasChanged(); - } - - private async Task ToggleJobs(long sessionId) - { - if (expandedSession == sessionId) - { - expandedSession = null; - jobs = null; - return; - } - jobs = await SessionRepo.GetJobsForSessionAsync(sessionId); - expandedSession = sessionId; - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor deleted file mode 100644 index 7c9b082..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor +++ /dev/null @@ -1,228 +0,0 @@ -@page "/config/{Username}" -@inject TwitchArchive.Core.Config.IConfigurationService ConfigService -@inject NavigationManager Nav - -

Streamer Config: @Username

- - - - - @if (saved) - { -
Saved.
- } -
- - -
- -
- - -
- -
- - -
- -
- - -
- - @* Streamlink path is global-only; not configurable per-streamer *@ - -
-

Per-streamer settings

-
- - -
-
- - -
-
- - -
-
- - - - - - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - -
-
- - - -
- -@* Confirmation modal *@ -@if (showConfirm) -{ - -} - -@code { - [Parameter] - public string Username { get; set; } = string.Empty; - private TwitchArchive.Core.Config.StreamerConfig model = new(); - private TwitchArchive.Core.Config.GlobalConfig? global; - private bool saved = false; - private bool showConfirm = false; - - // local values for nullable per-streamer settings (bind safely) - private bool downloadVODVal; - private bool downloadCHATVal; - private bool downloadLiveCHATVal; - private bool mergeVideoChatVal; - private string mergeChatLayoutVal = "side-by-side"; - private int? vodTimeoutVal; - private bool deleteFilesVal; - private bool uploadPreMergeVideoVal; - private bool uploadMergedVideoVal; - private bool uploadChatVideoVal; - private bool onlyRawVal; - private bool cleanRawVal; - private bool uploadToCloudVal; - - protected override void OnInitialized() - { - global = ConfigService.LoadGlobal(); - var s = ConfigService.LoadStreamer(Username); - var isNew = s == null; - if (s != null) model = s; - // initialize local values from model or global defaults - downloadVODVal = model.DownloadVOD ?? global?.Defaults.DownloadVOD ?? true; - downloadCHATVal = model.DownloadCHAT ?? global?.Defaults.DownloadCHAT ?? true; - downloadLiveCHATVal = model.DownloadLiveCHAT ?? global?.Defaults.DownloadLiveCHAT ?? true; - mergeVideoChatVal = model.MergeVideoChat ?? global?.Defaults.MergeVideoChat ?? false; - mergeChatLayoutVal = model.MergeChatLayout ?? global?.Defaults.MergeChatLayout ?? "side-by-side"; - vodTimeoutVal = model.VodTimeout ?? global?.Defaults.VodTimeout ?? 300; - deleteFilesVal = model.DeleteFiles ?? global?.Defaults.DeleteFiles ?? false; - uploadPreMergeVideoVal = model.UploadPreMergeVideo ?? global?.Defaults.UploadPreMergeVideo ?? true; - uploadMergedVideoVal = model.UploadMergedVideo ?? global?.Defaults.UploadMergedVideo ?? true; - uploadChatVideoVal = model.UploadChatVideo ?? global?.Defaults.UploadChatVideo ?? false; - onlyRawVal = model.OnlyRaw ?? global?.Defaults.OnlyRaw ?? false; - cleanRawVal = model.CleanRaw ?? global?.Defaults.CleanRaw ?? true; - uploadToCloudVal = model.UploadToCloud ?? global?.UploadToCloud ?? false; - - // when creating a new streamer config, populate model with global defaults so - // the streamer config stores initial values and subsequent edits use streamer values - if (isNew) - { - model.Quality = model.Quality ?? global?.DefaultQuality; - model.UploadToCloud = uploadToCloudVal; - model.UploadDestination = model.UploadDestination ?? global?.UploadDestination; - model.DownloadVOD = downloadVODVal; - model.DownloadCHAT = downloadCHATVal; - model.DownloadLiveCHAT = downloadLiveCHATVal; - model.MergeVideoChat = mergeVideoChatVal; - model.MergeChatLayout = mergeChatLayoutVal; - model.VodTimeout = vodTimeoutVal; - model.DeleteFiles = deleteFilesVal; - model.UploadPreMergeVideo = uploadPreMergeVideoVal; - model.UploadMergedVideo = uploadMergedVideoVal; - model.UploadChatVideo = uploadChatVideoVal; - model.OnlyRaw = onlyRawVal; - model.CleanRaw = cleanRawVal; - } - } - - private void Save() - { - model.Username = Username; - if (string.IsNullOrWhiteSpace(model.Quality)) model.Quality = null; - // Upload to cloud - model.UploadToCloud = uploadToCloudVal; - // Ensure global-only settings are not stored per-streamer - model.StreamlinkPath = null; - model.HlsSegments = null; - model.HlsSegmentsVOD = null; - model.StreamlinkTtvlol = null; - model.FfmpegHwaccel = null; - model.FfmpegThreads = null; - model.FfmpegAudioBitrate = null; - model.FfmpegAudioCodec = null; - model.FfmpegAudioSamplerate = null; - model.FfmpegErrorRecovery = null; - model.FfmpegFaststart = null; - model.FfmpegProgress = null; - // Per-streamer values: always map local values into the model - model.DownloadVOD = downloadVODVal; - model.DownloadCHAT = downloadCHATVal; - model.DownloadLiveCHAT = downloadLiveCHATVal; - model.MergeVideoChat = mergeVideoChatVal; - model.MergeChatLayout = mergeChatLayoutVal; - model.VodTimeout = vodTimeoutVal; - model.DeleteFiles = deleteFilesVal; - model.UploadPreMergeVideo = uploadPreMergeVideoVal; - model.UploadMergedVideo = uploadMergedVideoVal; - model.UploadChatVideo = uploadChatVideoVal; - model.OnlyRaw = onlyRawVal; - model.CleanRaw = cleanRawVal; - ConfigService.SaveStreamer(model); - saved = true; - } - - private void ConfirmDelete() - { - try - { - ConfigService.DeleteStreamer(Username); - showConfirm = false; - Nav.NavigateTo("/"); - } - catch - { - // ignore - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor deleted file mode 100644 index c9ac1e6..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor +++ /dev/null @@ -1,23 +0,0 @@ -@page "/streamer/{Username}" -@using TwitchArchive.Core.Workers -@inject StreamWorkerManager WorkerManager - -

Streamer: @Username

- -
- Status: @(WorkerManager.IsRunning(Username) ? "Live" : "Offline") - Edit Settings -
- -
-
Record
-
Process
-
Upload
-
- - - -@code { - [Parameter] - public string Username { get; set; } = string.Empty; -} diff --git a/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml deleted file mode 100644 index 1463d9e..0000000 --- a/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml +++ /dev/null @@ -1,19 +0,0 @@ -@page "/" -@namespace TwitchArchive.Web.Pages -@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers - - - - - - Twitch Archive - - - - - - - - - - diff --git a/dotnet/src/TwitchArchive.Web/Program.cs b/dotnet/src/TwitchArchive.Web/Program.cs deleted file mode 100644 index 12ddccb..0000000 --- a/dotnet/src/TwitchArchive.Web/Program.cs +++ /dev/null @@ -1,105 +0,0 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Authentication; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.EntityFrameworkCore; -using TwitchArchive.Core.Services; -using TwitchArchive.Core.Config; -using TwitchArchive.Core.Persistence; -using TwitchArchive.Web.Hubs; -using TwitchArchive.Web.Services; - -var builder = WebApplication.CreateBuilder(args); - -// Add services -builder.Services.AddRazorPages(); -builder.Services.AddServerSideBlazor(); -builder.Services.AddSignalR(); - -// Authentication -builder.Services.AddSingleton(); -builder.Services.AddAuthentication(options => -{ - options.DefaultScheme = Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme; -}) -.AddCookie(options => { options.LoginPath = "/login"; }); - -// Register Core services -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddHttpClient(client => { /* base config if needed */ }); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); - -// Configuration service for global + per-streamer JSON files in ./config -builder.Services.AddScoped(); - -// Broadcaster forwards output store events to the SignalR hub -builder.Services.AddSingleton(); - -// SQLite DB (file in app folder) -var conn = "Data Source=archive.db"; -// Provide a factory for creating DbContext instances for background work -builder.Services.AddDbContextFactory(opt => opt.UseSqlite(conn)); -// persistence -builder.Services.AddScoped(); - -// Session cache and background refresh -builder.Services.AddSingleton(); -builder.Services.AddHostedService(); - -var app = builder.Build(); - -if (!app.Environment.IsDevelopment()) -{ - app.UseExceptionHandler("/Error"); -} - -// Ensure DB schema exists (creates DB when missing) -using (var scope = app.Services.CreateScope()) -{ - var factory = scope.ServiceProvider.GetRequiredService>(); - try - { - using var db = factory.CreateDbContext(); - // Apply any pending EF migrations (creates/updates schema as needed) - db.Database.Migrate(); - } - catch { } -} - -app.UseStaticFiles(); -app.UseRouting(); - -app.UseAuthentication(); -app.UseAuthorization(); - -// login endpoint -app.MapPost("/auth/login", async (Microsoft.AspNetCore.Http.HttpContext http, TwitchArchive.Web.Services.IAuthService auth) => -{ - try - { - var form = await http.Request.ReadFormAsync(); - var pwd = form["password"].ToString(); - if (auth.ValidatePassword(pwd)) - { - var claims = new[] { new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "admin") }; - var id = new System.Security.Claims.ClaimsIdentity(claims, Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme); - var principal = new System.Security.Claims.ClaimsPrincipal(id); - await http.SignInAsync(Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme, principal); - http.Response.Redirect("/"); - return; - } - } - catch { } - http.Response.Redirect("/login?error=1"); -}); - -app.MapBlazorHub(); -app.MapHub("/processOutputHub"); -app.MapFallbackToPage("/_Host"); - -app.Run(); diff --git a/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json deleted file mode 100644 index 9c88f37..0000000 --- a/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "profiles": { - "TwitchArchive.Web": { - "commandName": "Project", - "launchBrowser": true, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:64466;http://localhost:64467" - } - } -} \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs deleted file mode 100644 index be83fc0..0000000 --- a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Linq; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using TwitchArchive.Core.Persistence; -using TwitchArchive.Core.Persistence.Models; - -namespace TwitchArchive.Web.Services -{ - public class AuthService : IAuthService - { - private readonly IDbContextFactory _dbFactory; - private readonly ILogger _log; - private string? _cachedHash; - - public AuthService(IDbContextFactory dbFactory, ILogger log) - { - _dbFactory = dbFactory ?? throw new ArgumentNullException(nameof(dbFactory)); - _log = log; - Refresh(); - } - - public void Refresh() - { - try - { - using var ctx = _dbFactory.CreateDbContext(); - var u = ctx.UserCredentials.AsNoTracking().FirstOrDefault(); - _cachedHash = u?.PasswordHash; - } - catch (Exception ex) - { - _log?.LogWarning(ex, "Failed to read password from database"); - _cachedHash = null; - } - } - - public bool ValidatePassword(string plain) - { - // If no password configured, allow access (default open) - if (string.IsNullOrWhiteSpace(_cachedHash)) return true; - try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _cachedHash); } - catch (Exception ex) - { - _log?.LogWarning(ex, "Password verification failed"); - return false; - } - } - - public void SetPasswordHash(string hash) - { - try - { - using var ctx = _dbFactory.CreateDbContext(); - var existing = ctx.UserCredentials.FirstOrDefault(); - if (existing == null) - { - existing = new UserCredential { PasswordHash = hash }; - ctx.UserCredentials.Add(existing); - } - else - { - existing.PasswordHash = hash; - ctx.UserCredentials.Update(existing); - } - ctx.SaveChanges(); - _cachedHash = hash; - } - catch (Exception ex) - { - _log?.LogError(ex, "Failed to save password to database"); - } - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs deleted file mode 100644 index cd3b0fd..0000000 --- a/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Threading.Tasks; - -namespace TwitchArchive.Web.Services -{ - public interface IAuthService - { - bool ValidatePassword(string plain); - void Refresh(); - void SetPasswordHash(string hash); - } -} diff --git a/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs deleted file mode 100644 index 641ddf9..0000000 --- a/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs +++ /dev/null @@ -1,35 +0,0 @@ -using Microsoft.AspNetCore.SignalR; -using System; -using TwitchArchive.Core.Services; -using TwitchArchive.Web.Hubs; - -namespace TwitchArchive.Web.Services -{ - public class ProcessOutputBroadcaster : IDisposable - { - private readonly IProcessOutputStore _store; - private readonly IHubContext _hub; - - public ProcessOutputBroadcaster(IProcessOutputStore store, IHubContext hub) - { - _store = store; - _hub = hub; - _store.LineAppended += OnLineAppended; - } - - private void OnLineAppended(string streamer, OutputLine line) - { - try - { - // send to clients; clients should listen on "ReceiveLine" - _hub.Clients.Group(streamer).SendAsync("ReceiveLine", streamer, line); - } - catch { } - } - - public void Dispose() - { - _store.LineAppended -= OnLineAppended; - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs deleted file mode 100644 index abe7855..0000000 --- a/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using TwitchArchive.Core.Persistence.Models; - -namespace TwitchArchive.Web.Services -{ - public class SessionCacheService - { - private readonly object _lock = new(); - private Dictionary _snapshot = new(); - - public event Action? Updated; - - public void Update(IEnumerable sessions) - { - var next = new Dictionary(); - foreach (var s in sessions) - { - if (!next.ContainsKey(s.StreamerUsername)) next[s.StreamerUsername] = s.StartedAt; - } - - lock (_lock) - { - _snapshot = next; - } - - Updated?.Invoke(); - } - - public Dictionary GetSnapshot() - { - lock (_lock) - { - return new Dictionary(_snapshot); - } - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs deleted file mode 100644 index 93523b8..0000000 --- a/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using TwitchArchive.Core.Persistence; - -namespace TwitchArchive.Web.Services -{ - public class SessionRefreshHostedService : BackgroundService - { - private readonly IServiceScopeFactory _scopeFactory; - private readonly SessionCacheService _cache; - private readonly ILogger _logger; - private readonly TimeSpan _interval; - - public SessionRefreshHostedService(IServiceScopeFactory scopeFactory, SessionCacheService cache, ILogger logger) - { - _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); - _cache = cache ?? throw new ArgumentNullException(nameof(cache)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _interval = TimeSpan.FromSeconds(10); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - using var timer = new PeriodicTimer(_interval); - try - { - while (await timer.WaitForNextTickAsync(stoppingToken)) - { - try - { - using var scope = _scopeFactory.CreateScope(); - var repo = scope.ServiceProvider.GetRequiredService(); - var sessions = await repo.GetRecentSessionsAsync(200, stoppingToken).ConfigureAwait(false); - _cache.Update(sessions); - } - catch (OperationCanceledException) { break; } - catch (Exception ex) - { - _logger.LogError(ex, "Error while refreshing sessions"); - } - } - } - catch (OperationCanceledException) { } - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor deleted file mode 100644 index cf6d00c..0000000 --- a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor +++ /dev/null @@ -1,31 +0,0 @@ -@inherits LayoutComponentBase -
-
- -

Twitch Archive

- -
-
-
- -
-
@Body
-
-
- -@code { - bool sidebarCollapsed; - void ToggleSidebar() => sidebarCollapsed = !sidebarCollapsed; -} diff --git a/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor deleted file mode 100644 index f5c2c76..0000000 --- a/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor +++ /dev/null @@ -1,93 +0,0 @@ -@using TwitchArchive.Core.Services -@using Microsoft.AspNetCore.Components -@using Microsoft.AspNetCore.SignalR.Client -@using System.Linq -@inject IProcessOutputStore OutputStore -@inject NavigationManager Navigation - -@inherits ComponentBase -
- @foreach (var line in lines) - { -
@line.TimestampUtc.ToLocalTime().ToString("HH:mm:ss") - @line.Line
- } -
- -@code { - [Parameter] - public string Streamer { get; set; } = string.Empty; - - private List lines = new(); - private HubConnection? hubConnection; - private string? _currentGroup; - - protected override async Task OnInitializedAsync() - { - OutputStore.LineAppended += OnLineAppended; - - hubConnection = new HubConnectionBuilder() - .WithUrl(Navigation.ToAbsoluteUri("/processOutputHub")) - .WithAutomaticReconnect() - .Build(); - - hubConnection.On("ReceiveLine", (streamer, line) => - { - if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; - // avoid duplicates - if (lines.Any(l => l.TimestampUtc == line.TimestampUtc && l.Line == line.Line)) return; - lines.Add(line); - if (lines.Count > 1000) lines.RemoveAt(0); - InvokeAsync(StateHasChanged); - }); - - await hubConnection.StartAsync(); - if (!string.IsNullOrWhiteSpace(Streamer)) - { - await hubConnection.SendAsync("JoinStreamerGroup", Streamer); - _currentGroup = Streamer; - } - - await base.OnInitializedAsync(); - } - - protected override async Task OnParametersSetAsync() - { - if (hubConnection == null) return; - if (!string.IsNullOrWhiteSpace(_currentGroup) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) - { - try { await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); } catch { } - _currentGroup = null; - } - - if (!string.IsNullOrWhiteSpace(Streamer) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) - { - try { await hubConnection.SendAsync("JoinStreamerGroup", Streamer); _currentGroup = Streamer; } catch { } - } - - await base.OnParametersSetAsync(); - } - - private void OnLineAppended(string streamer, OutputLine line) - { - if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; - lines.Add(line); - if (lines.Count > 1000) lines.RemoveAt(0); - InvokeAsync(StateHasChanged); - } - - public async ValueTask DisposeAsync() - { - OutputStore.LineAppended -= OnLineAppended; - if (hubConnection != null) - { - try - { - if (!string.IsNullOrWhiteSpace(_currentGroup)) await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); - } - catch { } - try { await hubConnection.StopAsync(); } catch { } - try { await hubConnection.DisposeAsync(); } catch { } - hubConnection = null; - } - } -} diff --git a/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj deleted file mode 100644 index 774299b..0000000 --- a/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/dotnet/src/TwitchArchive.Web/_Imports.razor b/dotnet/src/TwitchArchive.Web/_Imports.razor deleted file mode 100644 index 34b6706..0000000 --- a/dotnet/src/TwitchArchive.Web/_Imports.razor +++ /dev/null @@ -1,10 +0,0 @@ -@using System -@using System.Net.Http -@using Microsoft.AspNetCore.Components -@using Microsoft.AspNetCore.Components.Web -@using Microsoft.AspNetCore.Components.Routing -@using Microsoft.AspNetCore.Components.Forms -@using Microsoft.AspNetCore.Authorization -@using TwitchArchive.Core -@using TwitchArchive.Core.Services -@using TwitchArchive.Web.Shared diff --git a/dotnet/src/TwitchArchive.Web/archive.db b/dotnet/src/TwitchArchive.Web/archive.db deleted file mode 100644 index 4e86411..0000000 Binary files a/dotnet/src/TwitchArchive.Web/archive.db and /dev/null differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm deleted file mode 100644 index fd7d358..0000000 Binary files a/dotnet/src/TwitchArchive.Web/archive.db-shm and /dev/null differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-wal b/dotnet/src/TwitchArchive.Web/archive.db-wal deleted file mode 100644 index 4596f14..0000000 Binary files a/dotnet/src/TwitchArchive.Web/archive.db-wal and /dev/null differ diff --git a/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css deleted file mode 100644 index 5c247f8..0000000 --- a/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css +++ /dev/null @@ -1,34 +0,0 @@ -/* App layout styles for Twitch Archive */ -:root{ - --bg:#0f1720; --panel:#111322; --muted:#9aa4c5; --accent:#7dd3fc; --accent-2:#89b4fa; --card:#0b1220; -} -html,body { height:100%; margin:0; font-family:Segoe UI, Roboto, -apple-system, sans-serif; background:var(--bg); color:#e6eef8; } -.page { display:flex; min-height:100vh; } -.sidebar { width:220px; flex-shrink:0; background:var(--panel); color:var(--muted); overflow-y:auto; padding:1rem 0; box-shadow:2px 0 8px rgba(6,6,10,0.6); } -.sidebar.collapsed { width:66px; } -.brand { font-weight:600; padding:0 1rem 0.8rem 1rem; color:#fff; } -.nav-list { list-style:none; margin:0; padding:0; } -.nav-list li { margin:0.2rem 0; } -.nav-link { display:block; padding:0.6rem 1rem; color:var(--muted); text-decoration:none; border-left:4px solid transparent; } -.nav-link.active, .nav-link:hover { background:linear-gradient(90deg, rgba(255,255,255,0.02), transparent); color:#fff; border-left-color:var(--accent-2); } -.topbar { display:flex; align-items:center; background:var(--panel); padding:0.6rem 1rem; box-shadow:0 2px 6px rgba(0,0,0,0.4); } -.topbar .topbar-inner { display:flex; align-items:center; width:100%; } -.hamburger { font-size:1.2rem; margin-right:1rem; background:transparent; border:none; color:inherit; cursor:pointer; } -.title { margin:0; font-size:1.05rem; flex:1; } -.top-actions .action { color:var(--muted); text-decoration:none; margin-left:1rem; } -.main { flex:1; overflow-y:auto; padding:1.5rem; } -.container { max-width:1100px; margin:0 auto; } - -/* Cards and forms */ -.card { background:var(--card); padding:1rem; border-radius:8px; box-shadow:0 2px 8px rgba(2,6,23,0.6); margin-bottom:1rem; } -.card-header { display:flex; justify-content:space-between; align-items:center; font-weight:600; } -.card-body { margin-top:0.5rem; color:var(--muted); } -button { background:var(--accent); border:none; color:#002; padding:0.45rem 0.8rem; border-radius:6px; cursor:pointer; } -button:hover { opacity:0.95; } -input[type=password], input[type=text], input[type=number], .input, InputText { padding:0.5rem; border-radius:6px; border:1px solid rgba(255,255,255,0.06); background:transparent; color:inherit; } - -@media(max-width:768px) { - .sidebar { display:none; } - .topbar { display:flex; } -} - diff --git a/modules/constants.py b/modules/constants.py index 7aa10eb..7f09e35 100644 --- a/modules/constants.py +++ b/modules/constants.py @@ -32,9 +32,6 @@ DEFAULT_CONFIG = { 'mergeChatLayout': 'side-by-side', # Layout: 'side-by-side' or 'overlay' 'vodTimeout': 300, 'uploadCloud': True, - 'uploadPreMergeVideo': True, # Upload original videos before merging - 'uploadMergedVideo': True, # Upload merged videos (video + chat) - 'uploadChatVideo': False, # Upload standalone chat video 'deleteFiles': False, 'onlyRaw': False, 'cleanRaw': True, diff --git a/modules/downloader.py b/modules/downloader.py index 87fe30d..2507bd6 100644 --- a/modules/downloader.py +++ b/modules/downloader.py @@ -4,13 +4,10 @@ Includes fallback support for chat_downloader when VOD-based methods fail. """ import os -import sys import subprocess import json import threading import time -import socket -import re from typing import Dict, Any, Optional from colorama import Fore, Style @@ -41,15 +38,11 @@ class ContentDownloader: self.ffmpeg_path = ffmpeg_path self.quality = config.get('quality', 'best') self.hls_segments_vod = config.get('hls_segmentsVOD', 10) - self.download_vod_enabled = config.get('downloadVOD', True) - self.download_chat_enabled = config.get('downloadCHAT', True) - self.download_live_chat_enabled = config.get('downloadLiveCHAT', True) + self.download_vod = config.get('downloadVOD', True) + self.download_chat = config.get('downloadCHAT', True) + self.download_live_chat = config.get('downloadLiveCHAT', True) self.use_chat_downloader_primary = config.get('useChatDownloaderPrimary', False) self.use_chat_downloader_fallback = config.get('useChatDownloaderFallback', True) - default_chat_font = 'Arial' if sys.platform.startswith('win') else 'DejaVu Sans' - self.chat_render_font = config.get('chat_render_font', default_chat_font) - self.last_chat_render_attempted = False - self.last_chat_render_succeeded = False # Initialize chat_downloader if available self.chat_downloader = None @@ -66,11 +59,6 @@ class ContentDownloader: self.chat_thread = None self.chat_thread_success = False self.chat_thread_error = None - - def reset_chat_render_status(self) -> None: - """Reset chat render tracking before a processing pass.""" - self.last_chat_render_attempted = False - self.last_chat_render_succeeded = False def download_vod(self, vod_info: Dict[str, Any], output_path: str) -> bool: """ @@ -83,7 +71,7 @@ class ContentDownloader: Returns: bool: True if download succeeded, False otherwise """ - if not self.download_vod_enabled: + if not self.download_vod: return False print(f'\n{Fore.CYAN}Downloading VOD: {vod_info["title"]}{Style.RESET_ALL}') @@ -200,7 +188,7 @@ class ContentDownloader: '-h', '1080', '--framerate', '30', '--outline', - '-f', self.chat_render_font, + '-f', 'Arial', '--font-size', '22', '--update-rate', '1.0', '--offline', @@ -225,9 +213,6 @@ class ContentDownloader: try: print(f'{Fore.YELLOW}Rendering chat video...{Style.RESET_ALL}') - print(f'{Fore.CYAN}Using chat font: {self.chat_render_font}{Style.RESET_ALL}') - self.last_chat_render_attempted = True - self.last_chat_render_succeeded = False # Build complete command full_cmd = [self.twitch_downloader_path, 'chatrender', '-i', json_path, '-o', video_path] + chat_settings @@ -262,7 +247,6 @@ class ContentDownloader: print(f'{Fore.RED}✗ Chat video file is too small ({file_size} bytes){Style.RESET_ALL}') return False - self.last_chat_render_succeeded = True print(f'{Fore.GREEN}✓ Chat rendered ({file_size:,} bytes){Style.RESET_ALL}') return True @@ -286,7 +270,7 @@ class ContentDownloader: Returns: bool: True if succeeded, False otherwise """ - if not self.download_chat_enabled: + if not self.download_chat: return False print(f'\n{Fore.CYAN}Downloading chat: {vod_info["title"]}{Style.RESET_ALL}') @@ -312,7 +296,7 @@ class ContentDownloader: Returns: subprocess.Popen: The process handle, or None if failed to start """ - if not self.download_live_chat_enabled: + if not self.download_live_chat: return None print(f'\n{Fore.CYAN}Starting live chat download...{Style.RESET_ALL}') @@ -341,118 +325,6 @@ class ContentDownloader: except Exception as e: print(f'{Fore.RED}✗ Failed to start live chat download: {str(e)}{Style.RESET_ALL}') return None - - def _download_live_chat_via_irc(self, username: str, json_path: str, - max_messages: Optional[int] = None, - timeout: Optional[float] = None, - shutdown_check: Optional[callable] = None, - stream_monitor = None, - verbose: bool = False) -> bool: - """ - Simple IRC-based fallback to capture Twitch chat when GraphQL methods fail. - - This writes newline-delimited JSON objects with at least: timestamp (ms), - author (dict with `name`), and `message`. - """ - try: - sock = socket.socket() - sock.connect(('irc.chat.twitch.tv', 6667)) - sock.settimeout(1.0) - - # Request tags & capabilities - sock.sendall(b'CAP REQ :twitch.tv/tags twitch.tv/commands twitch.tv/membership\r\n') - sock.sendall(b'PASS SCHMOOPIIE\r\n') - sock.sendall(b'NICK justinfan67420\r\n') - sock.sendall(f'JOIN #{username}\r\n'.encode('utf-8')) - - messages_written = 0 - start_time = time.time() - - # Open file for streaming newline-delimited JSON - os.makedirs(os.path.dirname(json_path), exist_ok=True) - with open(json_path, 'w', encoding='utf-8') as out_f: - buffer = '' - while True: - # Shutdown/timeouts - if shutdown_check and shutdown_check(): - break - if timeout and (time.time() - start_time) > timeout: - break - if stream_monitor: - try: - if not stream_monitor.is_user_live(): - break - except Exception: - pass - - try: - data = sock.recv(4096).decode('utf-8', 'ignore') - except socket.timeout: - continue - except Exception as e: - print(f'{Fore.YELLOW}⚠ IRC recv error: {e}{Style.RESET_ALL}') - break - - if not data: - continue - - buffer += data - lines = buffer.split('\r\n') - buffer = lines.pop() # remainder - - for line in lines: - if not line: - continue - # Respond to PINGs - if line.startswith('PING'): - try: - sock.sendall(b'PONG :tmi.twitch.tv\r\n') - except Exception: - pass - continue - - # Extract PRIVMSG lines - m = re.match(r'(?:@[^ ]+ )?:([^!]+)!.* PRIVMSG #[^ ]+ :(.+)', line) - if not m: - continue - - author = m.group(1) - msg_text = m.group(2) - timestamp_ms = int(time.time() * 1000) - - item = { - 'timestamp': timestamp_ms, - 'author': {'name': author}, - 'message': msg_text - } - - out_f.write(json.dumps(item, ensure_ascii=False) + '\n') - out_f.flush() - messages_written += 1 - - if verbose and (messages_written % 10 == 0): - print(f'\n{Fore.GREEN}💬 {author}: {Fore.WHITE}{msg_text}{Style.RESET_ALL}') - - if max_messages and messages_written >= max_messages: - break - - if max_messages and messages_written >= max_messages: - break - - sock.close() - - if messages_written > 0: - print(f'\n{Fore.GREEN}✓ IRC fallback captured {messages_written} messages{Style.RESET_ALL}') - return True - else: - print(f'\n{Fore.RED}✗ IRC fallback captured no messages{Style.RESET_ALL}') - return False - - except Exception as e: - print(f'{Fore.RED}✗ IRC fallback failed: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - return False def wait_for_chat_download(self, process: Optional[subprocess.Popen], json_path: str, timeout: int = 300) -> bool: @@ -493,7 +365,6 @@ class ContentDownloader: max_messages: Optional[int] = None, timeout: Optional[float] = None, shutdown_check: Optional[callable] = None, - stream_monitor = None, verbose: bool = False) -> bool: """ Download live chat using chat_downloader library as fallback. @@ -505,7 +376,6 @@ class ContentDownloader: max_messages: Maximum messages to download (None = unlimited) timeout: Stop after this many seconds (None = until stream ends) shutdown_check: Optional callback function that returns True when shutdown requested - stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat message previews Returns: @@ -516,19 +386,9 @@ class ContentDownloader: print(f'{Fore.YELLOW} Install with: pip install chat-downloader{Style.RESET_ALL}') return False - if not self.download_live_chat_enabled: + if not self.download_live_chat: print(f'{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config{Style.RESET_ALL}') return False - - # If a stream monitor was provided, check that the user is currently live - if stream_monitor is not None: - try: - if not stream_monitor.is_user_live(): - print(f'{Fore.YELLOW}⚠ Stream is not live; skipping chat download{Style.RESET_ALL}') - return False - except Exception as e: - # If we couldn't determine live status, continue and let chat_downloader handle it - print(f'{Fore.YELLOW}⚠ Could not determine live status: {e} - proceeding with chat download{Style.RESET_ALL}') print(f'\n{Fore.CYAN}Starting live chat download (chat_downloader)...{Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader library version: {ChatDownloader.__module__}{Style.RESET_ALL}') @@ -541,53 +401,20 @@ class ContentDownloader: print(f'{Fore.MAGENTA}[VERBOSE] Timeout: {timeout}s (None = unlimited){Style.RESET_ALL}') print(f'{Fore.MAGENTA}[VERBOSE] Max messages: {max_messages} (None = unlimited){Style.RESET_ALL}') - # Get chat messages with a small retry loop to handle transient GQL/network issues + # Get chat messages print(f'{Fore.CYAN}Connecting to Twitch chat...{Style.RESET_ALL}') - chat = None - max_attempts = 3 - for attempt in range(1, max_attempts + 1): - try: - chat = self.chat_downloader.get_chat( - stream_url, - message_types=['text_message'], # Basic text messages - output=json_path, - timeout=timeout, - max_messages=max_messages - ) - break - except Exception as e: - # Provide a clearer, user-facing message for common failures - print(f"{Fore.YELLOW}⚠ chat_downloader attempt {attempt}/{max_attempts} failed: {str(e)}{Style.RESET_ALL}") - # On final attempt, dump traceback to help diagnose library internals - if attempt >= max_attempts: - print(f"{Fore.RED}✗ chat_downloader failed after {max_attempts} attempts. This may be caused by Twitch GraphQL changes or rate-limiting.{Style.RESET_ALL}") - print(f"{Fore.YELLOW} Try upgrading the chat-downloader package: pip install -U chat-downloader{Style.RESET_ALL}") - import traceback - traceback.print_exc() - # Try IRC fallback before giving up - print(f"{Fore.MAGENTA}[VERBOSE] Attempting IRC fallback for chat capture...{Style.RESET_ALL}") - try: - return self._download_live_chat_via_irc(username, json_path, - max_messages=max_messages, - timeout=timeout, - shutdown_check=shutdown_check, - stream_monitor=stream_monitor, - verbose=verbose) - except Exception as fallback_err: - print(f"{Fore.RED}✗ IRC fallback failed: {fallback_err}{Style.RESET_ALL}") - traceback.print_exc() - return False - else: - time.sleep(1) - continue + chat = self.chat_downloader.get_chat( + stream_url, + message_types=['text_message'], # Basic text messages + output=json_path, + timeout=timeout, + max_messages=max_messages + ) # The get_chat with output parameter writes to file automatically # We just need to iterate to trigger the download message_count = 0 - last_check_time = time.time() - check_interval = 10.0 # Check if stream is still live every 10 seconds - - print(f'{Fore.CYAN}Receiving chat messages (will stop when stream ends)...{Style.RESET_ALL}') + print(f'{Fore.CYAN}Receiving chat messages (press Ctrl+C to stop)...{Style.RESET_ALL}') try: for message in chat: # Check for shutdown request @@ -595,19 +422,6 @@ class ContentDownloader: print(f'\n{Fore.YELLOW}⚠ Chat download stopped by shutdown request{Style.RESET_ALL}') break - # Periodically check if stream is still live - current_time = time.time() - if stream_monitor and (current_time - last_check_time) >= check_interval: - last_check_time = current_time - try: - is_live = stream_monitor.is_user_live() - if not is_live: - print(f'\n{Fore.YELLOW}⚠ Stream ended, stopping chat download{Style.RESET_ALL}') - break - except Exception as check_error: - print(f'\n{Fore.YELLOW}⚠ Could not check stream status: {check_error}{Style.RESET_ALL}') - # Continue downloading to avoid false positives from API errors - message_count += 1 # Show progress every 100 messages @@ -653,7 +467,6 @@ class ContentDownloader: def start_chat_downloader_thread(self, username: str, json_path: str, shutdown_check: Optional[callable] = None, - stream_monitor = None, verbose: bool = False) -> threading.Thread: """ Start chat_downloader in a background thread. @@ -662,7 +475,6 @@ class ContentDownloader: username: Twitch username json_path: Path to save chat JSON shutdown_check: Callback to check for shutdown - stream_monitor: Optional stream monitor to check if stream is still live verbose: Show chat previews Returns: @@ -673,7 +485,6 @@ class ContentDownloader: self.chat_thread_success = self.download_live_chat_with_chat_downloader( username, json_path, shutdown_check=shutdown_check, - stream_monitor=stream_monitor, verbose=verbose ) except Exception as e: diff --git a/modules/file_manager.py b/modules/file_manager.py index b024e8f..8848b8a 100644 --- a/modules/file_manager.py +++ b/modules/file_manager.py @@ -9,7 +9,7 @@ import subprocess from typing import List from colorama import Fore, Style -from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA, PREFIX_MERGED +from .constants import PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_METADATA from .utils import get_bin_path @@ -28,9 +28,6 @@ class FileManager: self.root_path = pathlib.Path(root_path) self.username = username self.upload_cloud = config.get('uploadCloud', True) - self.upload_pre_merge_video = config.get('uploadPreMergeVideo', True) - self.upload_merged_video = config.get('uploadMergedVideo', True) - self.upload_chat_video = config.get('uploadChatVideo', True) self.delete_files = config.get('deleteFiles', False) self.clean_raw = config.get('cleanRaw', True) self.download_vod = config.get('downloadVOD', True) @@ -45,128 +42,6 @@ class FileManager: self.chat_mp4_path = self.root_path / username / "chat" self.metadata_path = self.root_path / username / "metadata" self.log_file = self.root_path / ".log" - - def _to_rclone_relative_path(self, *parts: str) -> str: - """Build a POSIX-style relative path for rclone --files-from.""" - return pathlib.PurePosixPath(*parts).as_posix() - - def _build_upload_relative_paths(self, filename_base: str) -> List[str]: - """Build the candidate upload list relative to root_path for rclone.""" - files_to_upload: List[str] = [ - self._to_rclone_relative_path(self.username, 'metadata', f"{PREFIX_METADATA}{filename_base}.json"), - self._to_rclone_relative_path(self.username, 'chat', 'json', f"{PREFIX_CHAT}{filename_base}.json") - ] - - if self.upload_pre_merge_video: - files_to_upload.extend([ - self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_LIVE}{filename_base}.ts"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp4"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_LIVE}{filename_base}.mp3"), - self._to_rclone_relative_path(self.username, 'video', 'raw', f"{PREFIX_VOD}{filename_base}.ts"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp4"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_VOD}{filename_base}.mp3") - ]) - - if self.upload_merged_video: - files_to_upload.extend([ - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp4"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{filename_base}.mp3"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4"), - self._to_rclone_relative_path(self.username, 'video', f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3") - ]) - - if self.upload_chat_video: - files_to_upload.append(self._to_rclone_relative_path(self.username, 'chat', f"{PREFIX_CHAT}{filename_base}.mp4")) - - return files_to_upload - - def _get_existing_upload_relative_paths(self, relative_paths: List[str]) -> List[str]: - """Filter candidate upload paths to the files that actually exist.""" - existing_paths: List[str] = [] - for relative_path in relative_paths: - if (self.root_path / pathlib.PurePosixPath(relative_path)).exists(): - existing_paths.append(relative_path) - return existing_paths - - def _run_rclone_copy(self, relative_paths: List[str], description: str) -> bool: - """Run rclone copy for a set of paths relative to root_path.""" - existing_paths = self._get_existing_upload_relative_paths(relative_paths) - missing_paths = [path for path in relative_paths if path not in existing_paths] - - if not existing_paths: - print(f'{Fore.RED}✗ Upload skipped: no matching files found for {description}{Style.RESET_ALL}') - for missing_path in missing_paths: - print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') - return False - - if missing_paths: - print(f'{Fore.YELLOW}⚠ Some configured upload files were not found and will be skipped{Style.RESET_ALL}') - for missing_path in missing_paths: - print(f'{Fore.YELLOW} Missing: {missing_path}{Style.RESET_ALL}') - - print(f'{Fore.CYAN}rclone source: {self.root_path.resolve()}{Style.RESET_ALL}') - print(f'{Fore.CYAN}rclone destination: {self.rclone_path}{Style.RESET_ALL}') - print(f'{Fore.CYAN}Files queued for upload: {len(existing_paths)}{Style.RESET_ALL}') - - bin_path = get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - with open(upload_list_path, 'w', encoding='utf-8', newline='\n') as f: - f.write('\n'.join(existing_paths)) - f.write('\n') - - try: - cmd = [ - 'rclone', 'copy', - str(self.root_path.resolve()), - self.rclone_path, - '--files-from', upload_list_path, - '--progress' - ] - - print(f'{Fore.CYAN}Running: {' '.join(cmd)}{Style.RESET_ALL}') - - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() - return proc.returncode == 0 - finally: - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - def run_rclone_smoke_test(self) -> bool: - """Create and upload a tiny metadata file to verify rclone output and configuration.""" - smoke_name = 'RCLONE_SMOKE_TEST' - smoke_relative_path = self._to_rclone_relative_path( - self.username, - 'metadata', - f"{PREFIX_METADATA}{smoke_name}.json" - ) - smoke_file_path = self.root_path / pathlib.PurePosixPath(smoke_relative_path) - - smoke_payload = { - 'type': 'rclone_smoke_test', - 'username': self.username - } - - smoke_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(smoke_file_path, 'w', encoding='utf-8') as f: - json.dump(smoke_payload, f, indent=2) - - print(f'{Fore.CYAN}Created smoke-test file: {smoke_file_path}{Style.RESET_ALL}') - try: - result = self._run_rclone_copy([smoke_relative_path], 'rclone smoke test') - if result: - print(f'{Fore.GREEN}✓ Rclone smoke test completed{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Rclone smoke test failed{Style.RESET_ALL}') - return result - finally: - if smoke_file_path.exists(): - smoke_file_path.unlink() def initialize_directories(self) -> None: """Create all necessary directory structures.""" @@ -242,25 +117,55 @@ class FileManager: print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') if notification_callback: notification_callback(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - files_to_upload = self._build_upload_relative_paths(filename_base) - + + # Create list of files to upload + bin_path = get_bin_path() + upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') + + # Ensure temp directory exists + os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) + + files_to_upload = [ + f"{PREFIX_LIVE}{filename_base}.ts", + f"{PREFIX_LIVE}{filename_base}.mp4", + f"{PREFIX_LIVE}{filename_base}.mp3", + f"{PREFIX_VOD}{filename_base}.ts", + f"{PREFIX_VOD}{filename_base}.mp4", + f"{PREFIX_VOD}{filename_base}.mp3", + f"{PREFIX_METADATA}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.json", + f"{PREFIX_CHAT}{filename_base}.mp4" + ] + + with open(upload_list_path, 'w') as f: + f.write('\n'.join(files_to_upload)) + + # Run rclone try: - result = self._run_rclone_copy(files_to_upload, f'archive batch {filename_base}') - - if result: + result = subprocess.call([ + 'rclone', 'copy', + str(self.root_path.resolve()), + self.rclone_path, + '--include-from', upload_list_path + ]) + + # Clean up upload list + if os.path.exists(upload_list_path): + os.remove(upload_list_path) + + if result == 0: print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') if notification_callback: notification_callback(f'✓ Upload Success - {filename_base}', 'All files uploaded successfully') return True - - print(f'{Fore.RED}✗ Upload failed{Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - if notification_callback: - notification_callback(f'✗ Upload Failed - {filename_base}', - 'Upload failed. Files preserved locally. Check rclone output above.') - return False - + else: + print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') + print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') + if notification_callback: + notification_callback(f'✗ Upload Failed - {filename_base}', + f'Upload failed with code {result}. Files preserved locally.') + return False + except Exception as e: print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') return False @@ -270,8 +175,6 @@ class FileManager: """ Delete local archive files after successful upload. - Only deletes files that were configured to be uploaded. - Args: filename_base: Base filename (without prefixes/extensions) live_raw_path: Path to live raw file @@ -288,15 +191,14 @@ class FileManager: files_to_delete: List[str] = [] - # Live files (only if pre-merge videos are uploaded) - if self.upload_pre_merge_video: - if not self.clean_raw and os.path.exists(live_raw_path): - files_to_delete.append(live_raw_path) - if os.path.exists(live_proc_path): - files_to_delete.append(live_proc_path) + # Live files + if not self.clean_raw and os.path.exists(live_raw_path): + files_to_delete.append(live_raw_path) + if os.path.exists(live_proc_path): + files_to_delete.append(live_proc_path) - # VOD files (only if pre-merge videos are uploaded) - if self.download_vod and self.upload_pre_merge_video: + # VOD files + if self.download_vod: vod_raw = self.raw_path / f"{PREFIX_VOD}{filename_base}.ts" vod_mp4 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp4" vod_mp3 = self.video_path / f"{PREFIX_VOD}{filename_base}.mp3" @@ -308,37 +210,17 @@ class FileManager: if vod_mp3.exists(): files_to_delete.append(str(vod_mp3)) - # Merged video files (only if merged videos are uploaded) - if self.upload_merged_video: - merged_live_mp4 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp4" - merged_live_mp3 = self.video_path / f"{PREFIX_MERGED}{filename_base}.mp3" - merged_vod_mp4 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp4" - merged_vod_mp3 = self.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}.mp3" - - if merged_live_mp4.exists(): - files_to_delete.append(str(merged_live_mp4)) - if merged_live_mp3.exists(): - files_to_delete.append(str(merged_live_mp3)) - if merged_vod_mp4.exists(): - files_to_delete.append(str(merged_vod_mp4)) - if merged_vod_mp3.exists(): - files_to_delete.append(str(merged_vod_mp3)) - # Chat files if self.download_chat: chat_json = self.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json" + chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" - # Always delete JSON (it's always uploaded) if chat_json.exists(): files_to_delete.append(str(chat_json)) - - # Only delete chat MP4 if chat videos are uploaded - if self.upload_chat_video: - chat_mp4 = self.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4" - if chat_mp4.exists(): - files_to_delete.append(str(chat_mp4)) + if chat_mp4.exists(): + files_to_delete.append(str(chat_mp4)) - # Metadata files (always uploaded) + # Metadata files if self.download_metadata: metadata = self.metadata_path / f"{PREFIX_METADATA}{filename_base}.json" if metadata.exists(): diff --git a/modules/processor.py b/modules/processor.py index 58bd47f..b1cae71 100644 --- a/modules/processor.py +++ b/modules/processor.py @@ -6,7 +6,7 @@ import os import subprocess from colorama import Fore, Style -from .utils import detect_hardware_acceleration, get_hwaccel_encoder, resolve_hwaccel_type +from .utils import detect_hardware_acceleration, get_hwaccel_encoder class StreamProcessor: @@ -36,80 +36,38 @@ class StreamProcessor: config.get('ffmpeg_hwaccel', 'auto'), os_type ) - self.hwaccel_type = resolve_hwaccel_type(self.hwaccel_type, os_type) - def process_raw_stream(self, raw_path: str, output_path: str) -> bool: + def process_raw_stream(self, raw_path: str, output_path: str) -> None: """ Process raw .ts file into mp4/mp3 using ffmpeg. Args: raw_path: Path to the raw .ts file output_path: Path for the processed output file - - Returns: - bool: True when conversion succeeded, False otherwise """ if not os.path.exists(raw_path): print(f'{Fore.YELLOW}⚠ Raw file not found, skipping processing{Style.RESET_ALL}') - return False + return if self.only_raw: print(f'{Fore.CYAN}Keeping raw .ts file (onlyRaw mode){Style.RESET_ALL}') - return False + return print(f'{Fore.YELLOW}Processing raw stream file...{Style.RESET_ALL}') # Build ffmpeg command based on quality if self.quality == 'audio_only': - result = self._process_audio(raw_path, output_path) + self._process_audio(raw_path, output_path) else: - result = self._process_video(raw_path, output_path) + self._process_video(raw_path, output_path) - if result: - print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Stream processing failed{Style.RESET_ALL}') - - return result + print(f'{Fore.GREEN}✓ Stream processed successfully{Style.RESET_ALL}') - def _run_ffmpeg_command(self, cmd: list, output_path: str) -> bool: - """Run FFmpeg while streaming its output to the terminal.""" - print(f'{Fore.CYAN}Running FFmpeg: {' '.join(cmd)}{Style.RESET_ALL}') - - process = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - encoding='utf-8', - errors='replace' - ) - - if process.stdout: - for line in process.stdout: - print(line, end='') - - result = process.wait() - if result != 0: - print(f'{Fore.RED}✗ FFmpeg exited with code: {result}{Style.RESET_ALL}') - return False - - if not os.path.exists(output_path): - print(f'{Fore.RED}✗ FFmpeg did not create output: {output_path}{Style.RESET_ALL}') - return False - - if os.path.getsize(output_path) == 0: - print(f'{Fore.RED}✗ FFmpeg created an empty output file: {output_path}{Style.RESET_ALL}') - return False - - return True - - def _process_audio(self, raw_path: str, output_path: str) -> bool: + def _process_audio(self, raw_path: str, output_path: str) -> None: """Process audio-only stream.""" # Audio-only conversion with modern AAC encoding cmd = [ self.ffmpeg_path, - '-y', '-i', raw_path, '-vn', # No video '-c:a', self.ffmpeg_audio_codec, @@ -127,9 +85,14 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - return self._run_ffmpeg_command(cmd, output_path) + + # Run FFmpeg + if self.ffmpeg_progress: + subprocess.call(cmd) + else: + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - def _process_video(self, raw_path: str, output_path: str) -> bool: + def _process_video(self, raw_path: str, output_path: str) -> None: """Process video stream.""" cmd = [ self.ffmpeg_path, @@ -172,7 +135,12 @@ class StreamProcessor: cmd.extend(['-movflags', '+faststart']) cmd.append(output_path) - return self._run_ffmpeg_command(cmd, output_path) + + # Run FFmpeg + if self.ffmpeg_progress: + subprocess.call(cmd) + else: + subprocess.call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) def build_chat_output_args(self) -> str: """ diff --git a/modules/recorder.py b/modules/recorder.py index 3518591..fa9c9d1 100644 --- a/modules/recorder.py +++ b/modules/recorder.py @@ -7,8 +7,6 @@ import subprocess from typing import Dict, Any, Optional from colorama import Fore, Style -from .utils import get_env_value - class StreamRecorder: """Handles live stream recording using streamlink.""" @@ -70,7 +68,7 @@ class StreamRecorder: print(f'{Fore.YELLOW} Consider disabling streamlink_ttvlol in config or using alternative methods{Style.RESET_ALL}') # Add authentication if available - oauth_token = get_env_value("OAUTH-PRIVATE-TOKEN", "OAUTH_PRIVATE_TOKEN", default="") + oauth_token = os.getenv("OAUTH-PRIVATE-TOKEN", "") if oauth_token and oauth_token != "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": cmd.extend(['--twitch-api-header', f'Authorization=OAuth {oauth_token}']) diff --git a/modules/stream_monitor.py b/modules/stream_monitor.py index a6b89be..6c1607a 100644 --- a/modules/stream_monitor.py +++ b/modules/stream_monitor.py @@ -9,7 +9,6 @@ import requests from colorama import Fore, Style from .constants import TWITCH_OAUTH_URL, TWITCH_API_URL, TWITCH_GQL_URL, TWITCH_GQL_CLIENT_ID -from .utils import get_env_value class StreamMonitor: @@ -41,9 +40,7 @@ class StreamMonitor: return self._oauth_token try: - client_id = get_env_value('CLIENT-ID', 'CLIENT_ID') - client_secret = get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') - url = f"{TWITCH_OAUTH_URL}?client_id={client_id}&client_secret={client_secret}&grant_type=client_credentials" + url = f"{TWITCH_OAUTH_URL}?client_id={os.getenv('CLIENT-ID')}&client_secret={os.getenv('CLIENT-SECRET')}&grant_type=client_credentials" response = requests.post(url, timeout=15) response.raise_for_status() self._oauth_token = response.json()['access_token'] @@ -72,7 +69,7 @@ class StreamMonitor: url = f'{TWITCH_API_URL}/users?login={self.username}' headers = { "Authorization": f"Bearer {self.get_oauth_token()}", - "Client-ID": get_env_value('CLIENT-ID', 'CLIENT_ID') + "Client-ID": os.getenv('CLIENT-ID') } response = requests.get(url, headers=headers, timeout=15) response.raise_for_status() @@ -118,34 +115,6 @@ class StreamMonitor: print(f'{Fore.YELLOW} {str(e)}{Style.RESET_ALL}') sys.exit(1) - def is_user_live(self) -> bool: - """ - Check if the configured user is currently live. - - Returns: - bool: True if user is live, False if offline - - Raises: - Exception: If API request fails (caller should handle) - """ - query = f'query{{user(login: "{self.username}") {{stream{{id title}}}}}}' - - try: - response = requests.post( - TWITCH_GQL_URL, - json={'query': query}, - headers={"Client-ID": TWITCH_GQL_CLIENT_ID}, - timeout=15 - ) - response.raise_for_status() - data = response.json() - stream_data = data.get('data', {}).get('user', {}).get('stream') - return stream_data is not None - - except requests.exceptions.RequestException as e: - # Don't exit, let caller handle this - raise Exception(f"Failed to check if user is live: {str(e)}") - def get_latest_vod(self) -> Optional[Dict[str, Any]]: """ Get the most recent VOD for the configured user. diff --git a/modules/utils.py b/modules/utils.py index 4e3b9b1..280afa8 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -4,7 +4,6 @@ Utility functions and helpers for Twitch Archive. import os import sys -import shutil import pathlib import subprocess from typing import Optional @@ -36,15 +35,6 @@ def get_bin_path() -> str: return str(pathlib.Path(__file__).parent.parent.resolve() / "bin") -def get_env_value(*names: str, default: Optional[str] = None) -> Optional[str]: - """Return the first non-empty environment variable from the provided names.""" - for name in names: - value = os.getenv(name) - if value not in (None, ""): - return value - return default - - def get_ffmpeg_executable(os_type: str) -> str: """ Get the platform-specific ffmpeg executable path. @@ -58,11 +48,6 @@ def get_ffmpeg_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'ffmpeg.exe') - - system_ffmpeg = shutil.which('ffmpeg') - if system_ffmpeg: - return system_ffmpeg - return os.path.join(bin_path, 'ffmpeg') @@ -79,11 +64,6 @@ def get_twitch_downloader_executable(os_type: str) -> str: bin_path = get_bin_path() if os_type == 'windows': return os.path.join(bin_path, 'TwitchDownloaderCLI.exe') - - system_twitch_downloader = shutil.which('TwitchDownloaderCLI') - if system_twitch_downloader: - return system_twitch_downloader - return os.path.join(bin_path, 'TwitchDownloaderCLI') @@ -184,24 +164,6 @@ def verify_streamlink() -> bool: return False -def verify_rclone() -> bool: - """Verify that rclone is available on PATH.""" - try: - result = subprocess.run(['rclone', 'version'], - capture_output=True, - text=True, - timeout=5) - if result.returncode == 0: - version_line = result.stdout.strip().splitlines()[0] if result.stdout.strip() else 'unknown' - print(f'{Fore.GREEN}✓ Rclone found ({version_line}){Style.RESET_ALL}') - return True - raise FileNotFoundError() - except (FileNotFoundError, subprocess.TimeoutExpired, IndexError): - print(f'{Fore.RED}✗ ERROR: rclone not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Install rclone and ensure it is on PATH{Style.RESET_ALL}') - return False - - def verify_ffmpeg(os_type: str) -> bool: """ Verify that ffmpeg is available. @@ -269,57 +231,20 @@ def detect_hardware_acceleration(hwaccel_config: str, os_type: str) -> Optional[ if hwaccel_config in ['nvenc', 'qsv', 'amf', 'vaapi']: return hwaccel_config - # Auto-detect: choose only hardware we can reasonably prove is present. + # Auto-detect: try to determine available hardware if hwaccel_config == 'auto': - if is_nvidia_runtime_available(): - return 'nvenc' - if is_vaapi_runtime_available(): - return 'vaapi' - return 'none' + # On Windows, NVIDIA is most common + if os_type == 'windows': + # Could check for nvidia-smi, but just return 'auto' for ffmpeg to decide + return 'auto' + else: + # On Linux, VAAPI is common for Intel/AMD, or NVENC for NVIDIA + # Let ffmpeg auto-detect + return 'auto' return None -def is_nvidia_runtime_available() -> bool: - """Return True when the current runtime appears to expose an NVIDIA GPU.""" - visible_devices = os.getenv('NVIDIA_VISIBLE_DEVICES', '').strip().lower() - if visible_devices in {'void', 'none'}: - return False - if visible_devices and visible_devices != 'all': - return True - - if shutil.which('nvidia-smi'): - return True - - return any( - os.path.exists(device_path) - for device_path in ('/dev/nvidiactl', '/dev/nvidia0', '/dev/nvidia-modeset') - ) - - -def is_vaapi_runtime_available() -> bool: - """Return True when Linux VAAPI render nodes are present.""" - return any( - os.path.exists(device_path) - for device_path in ('/dev/dri/renderD128', '/dev/dri/card0') - ) - - -def resolve_hwaccel_type(hwaccel_type: Optional[str], os_type: str) -> Optional[str]: - """Return a safe hardware acceleration choice for the current runtime.""" - if hwaccel_type in (None, 'none'): - return 'none' - - if hwaccel_type == 'nvenc': - return 'nvenc' if is_nvidia_runtime_available() else 'none' - - if hwaccel_type == 'vaapi': - return 'vaapi' if is_vaapi_runtime_available() else 'none' - - # Leave explicit QSV/AMF unchanged for non-container users; container auto-detect no longer picks them blindly. - return hwaccel_type - - def get_hwaccel_encoder(hwaccel_type: str) -> str: """ Get the appropriate hardware-accelerated encoder for the given acceleration type. diff --git a/only-vod-chat.py b/only-vod-chat.py index 4d9a92d..b3a948d 100644 --- a/only-vod-chat.py +++ b/only-vod-chat.py @@ -228,11 +228,7 @@ class TwitchArchive: elif self.os == 'linux': subprocess.call([bin_path+"/TwitchDownloaderCLI", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"]) if self.username == 'KalathrasLolweapon': print('Uploading html chat to b2 bucket') - proc = subprocess.Popen(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() + subprocess.call(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress']) except Exception as e: print('A ERROR has ocurred and chat will need to be updated to html manually') @@ -240,22 +236,9 @@ class TwitchArchive: print('Uploading files:') if self.os == 'windows': if self.username == 'KalathrasLolweapon': - proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() - proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() - else: - proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - if proc.stdout: - for line in proc.stdout: - print(line, end='') - proc.wait() + subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) + subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress']) + else:subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) elif self.os == 'linux':subprocess.call([bin_path+'/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username]) if self.deleteFiles == 1: diff --git a/run_chat_only.py b/run_chat_only.py deleted file mode 100644 index 5134f38..0000000 --- a/run_chat_only.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -""" -Start chat downloader standalone for testing without recording video. - -Usage: - python run_chat_only.py --username vinesauce [--output path] [--max-messages N] [--timeout S] [--verbose] - -This script uses the project's `ConfigManager` and `FileManager` to create -appropriate directories and then starts the chat downloader in a background -thread. Press Ctrl+C to stop. -""" -import argparse -import time -from datetime import datetime -import os - -from colorama import Fore, Style - -from modules.config import ConfigManager -from modules.file_manager import FileManager -from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_operating_system -from modules.downloader import ContentDownloader -from modules.stream_monitor import StreamMonitor - - -def main(): - parser = argparse.ArgumentParser(description='Run chat downloader standalone for testing') - parser.add_argument('--username', '-u', required=True, help='Twitch username/channel name') - parser.add_argument('--output', '-o', help='Output JSON path (optional)') - parser.add_argument('--max-messages', type=int, default=None, help='Max messages to capture') - parser.add_argument('--timeout', type=float, default=None, help='Timeout in seconds') - parser.add_argument('--verbose', action='store_true', help='Show verbose/chat previews') - parser.add_argument('--foreground', action='store_true', help='Run downloader in foreground (blocking)') - parser.add_argument('--use-chat-downloader-primary', action='store_true', help='Use chat_downloader as primary method') - parser.add_argument('--use-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_true', help='Allow chat_downloader as fallback (default)') - parser.add_argument('--no-chat-downloader-fallback', dest='use_chat_downloader_fallback', action='store_false', help='Disable chat_downloader fallback') - args = parser.parse_args() - - cfg = ConfigManager() - config = cfg.load_streamer_config(args.username) - - # Apply overrides from CLI - if args.use_chat_downloader_primary: - config['useChatDownloaderPrimary'] = True - if args.use_chat_downloader_fallback is not None: - config['useChatDownloaderFallback'] = bool(args.use_chat_downloader_fallback) - - # Ensure directories exist (use configured archive root path) - fm = FileManager(root_path=config.get('root_path', 'archive'), username=args.username, config=config) - fm.initialize_directories() - - # Build default output path if not provided - if args.output: - json_path = args.output - else: - ts = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss') - json_path = str(fm.chat_json_path / f"CHAT_TEST_{args.username}_{ts}.json") - - # Initialize downloader - os_type = detect_operating_system() - twitch_downloader_path = get_twitch_downloader_executable(os_type) - ffmpeg_path = get_ffmpeg_executable(os_type) - - downloader = ContentDownloader(twitch_downloader_path=twitch_downloader_path, - ffmpeg_path=ffmpeg_path, - config=config) - - print(f"{Fore.CYAN}Starting standalone chat downloader for {args.username}{Style.RESET_ALL}") - print(f"Output path: {json_path}") - - stop_requested = {'stop': False} - - def shutdown_check(): - return stop_requested['stop'] - - # Prepare stream monitor - stream_monitor = StreamMonitor(args.username) - - # If chat downloads are disabled in config, enter monitoring mode instead - if not downloader.download_live_chat: - print(f"{Fore.YELLOW}⚠ downloadLiveCHAT is disabled in config - entering monitoring mode for {args.username}{Style.RESET_ALL}") - try: - while True: - try: - is_live = stream_monitor.is_user_live() - if is_live: - print(f"{Fore.GREEN}✓ {args.username} is live! Exiting monitor. Run the archiver to record video.{Style.RESET_ALL}") - break - else: - print(f"{Fore.CYAN}{args.username} is offline - checking again in 30s...{Style.RESET_ALL}") - except Exception as e: - print(f"{Fore.YELLOW}⚠ Could not check stream status: {e}{Style.RESET_ALL}") - time.sleep(30) - except KeyboardInterrupt: - print('\nKeyboard interrupt received; stopping monitor...') - return - - # If chat download is enabled, but the stream is currently offline, wait until it goes live - try: - try: - if not stream_monitor.is_user_live(): - print(f"{Fore.CYAN}{args.username} is currently offline - waiting for live stream to start...{Style.RESET_ALL}") - while not stream_monitor.is_user_live(): - time.sleep(10) - except Exception: - # If we cannot determine live status, proceed to start chat downloader anyway - pass - - except KeyboardInterrupt: - print('\nKeyboard interrupt received; exiting...') - return - - # Start thread (stream_monitor passed so downloader can stop when stream ends) - thread = downloader.start_chat_downloader_thread( - args.username, - json_path, - shutdown_check=shutdown_check, - stream_monitor=stream_monitor, - verbose=args.verbose - ) - - try: - if args.foreground: - # Run download directly in foreground - print('Running in foreground; this will block until download completes or interrupted') - success = downloader.download_live_chat_with_chat_downloader( - args.username, - json_path, - max_messages=args.max_messages, - timeout=args.timeout, - shutdown_check=shutdown_check, - stream_monitor=None, - verbose=args.verbose - ) - print('Done, success=' + str(success)) - else: - # Wait for thread to finish or until interrupted - while thread.is_alive(): - time.sleep(0.5) - except KeyboardInterrupt: - print('\nKeyboard interrupt received; stopping downloader...') - stop_requested['stop'] = True - thread.join(timeout=5) - - -if __name__ == '__main__': - main() diff --git a/run_tests.ps1 b/run_tests.ps1 deleted file mode 100644 index f547251..0000000 --- a/run_tests.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# PowerShell script to run Twitch Archive unit tests -# Run this script to execute all unit tests - -Write-Host "======================================================================" -ForegroundColor Cyan -Write-Host "TWITCH ARCHIVE - Running Unit Tests" -ForegroundColor Cyan -Write-Host "======================================================================" -ForegroundColor Cyan -Write-Host "" - -# Check if virtual environment exists and activate it -$venvPath = ".\venv314\Scripts\Activate.ps1" -if (Test-Path $venvPath) { - Write-Host "✓ Activating virtual environment..." -ForegroundColor Green - & $venvPath -} else { - Write-Host "⚠ Virtual environment not found at $venvPath" -ForegroundColor Yellow - Write-Host " Continuing with system Python..." -ForegroundColor Yellow -} - -Write-Host "" - -# Run the tests -Write-Host "Running unit tests..." -ForegroundColor Cyan -python test_twitch_archive_simple.py - -# Check exit code -if ($LASTEXITCODE -eq 0) { - Write-Host "" - Write-Host "======================================================================" -ForegroundColor Green - Write-Host "✓ ALL TESTS PASSED" -ForegroundColor Green - Write-Host "======================================================================" -ForegroundColor Green -} else { - Write-Host "" - Write-Host "======================================================================" -ForegroundColor Red - Write-Host "✗ SOME TESTS FAILED" -ForegroundColor Red - Write-Host "======================================================================" -ForegroundColor Red - exit $LASTEXITCODE -} diff --git a/start_chat_only.bat b/start_chat_only.bat deleted file mode 100644 index e1c4d4a..0000000 --- a/start_chat_only.bat +++ /dev/null @@ -1,18 +0,0 @@ -rem @echo off -setlocal - -rem Set the path to your virtual environment -set VENV_PATH=.\venv314 - -rem Activate the virtual environment -call "%VENV_PATH%\Scripts\activate.bat" -rem Ensure required packages are installed -pip install -r requirements.txt - -rem Run the standalone chat downloader script -rem Usage: start_chat_only.bat [--output path] [--max-messages N] [--timeout S] [--verbose] [--foreground] [--use-chat-downloader-primary] -rem Pass username as -u and forward additional arguments (mirrors start.bat behavior) -python run_chat_only.py -u %1 %2 %3 %4 %5 %6 %7 %8 %9 - -rem Deactivate the virtual environment -call "%VENV_PATH%\Scripts\deactivate.bat" diff --git a/test_twitch_archive.py b/test_twitch_archive.py deleted file mode 100644 index 4432d81..0000000 --- a/test_twitch_archive.py +++ /dev/null @@ -1,592 +0,0 @@ -""" -Unit tests for Twitch Archive command-line options and configuration. - -Tests focus on: -- Command-line argument parsing (via getopt simulation) -- Options and option combinations -- Configuration loading and merging -- Mode selection logic - -Excludes actual download/processing functionality. - -To run these tests: - python test_twitch_archive.py - or - python -m pytest test_twitch_archive.py -v -""" - -import unittest -import sys -import os -import json -import tempfile -import shutil -import getopt -from unittest.mock import patch, MagicMock, Mock -from pathlib import Path - -# Add parent directory to path for imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from modules.config import ConfigManager -from modules.constants import DEFAULT_CONFIG - - -class TestCommandLineArgumentParsing(unittest.TestCase): - """Test command-line argument parsing logic using getopt directly.""" - - def setUp(self): - """Set up test fixtures.""" - self.test_dir = tempfile.mkdtemp() - self.original_cwd = os.getcwd() - os.chdir(self.test_dir) - - def tearDown(self): - """Clean up test fixtures.""" - os.chdir(self.original_cwd) - shutil.rmtree(self.test_dir, ignore_errors=True) - - def test_help_short_option(self): - """Test -h option parsing.""" - argv = ['-h'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - # Should parse successfully - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0][0], '-h') - - def test_help_long_option(self): - """Test --help option parsing.""" - argv = ['--help'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0][0], '--help') - - def test_username_short_option(self): - """Test -u username option parsing.""" - argv = ['-u', 'teststreamer'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('-u', 'teststreamer')) - - def test_username_long_option(self): - """Test --username option parsing.""" - argv = ['--username', 'teststreamer'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--username', 'teststreamer')) - - def test_verbose_option(self): - """Test --verbose option parsing.""" - argv = ['--verbose'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--verbose', '')) - - def test_chat_only_option(self): - """Test --chat-only option parsing.""" - argv = ['--chat-only'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--chat-only', '')) - - def test_legacy_option(self): - """Test --legacy option parsing.""" - argv = ['--legacy'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--legacy', '')) - - def test_chat_downloader_options(self): - """Test chat downloader option parsing.""" - argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 2) - self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) - self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) - - def test_legacy_quality_option(self): - """Test -q quality option parsing.""" - argv = ['-q', '720p'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('-q', '720p')) - - def test_legacy_boolean_options(self): - """Test legacy boolean option parsing.""" - argv = ['-v', '1', '-c', '0', '-m', '1'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 3) - self.assertEqual(opts[0], ('-v', '1')) - self.assertEqual(opts[1], ('-c', '0')) - self.assertEqual(opts[2], ('-m', '1')) - - def test_invalid_option(self): - """Test that invalid option raises error.""" - argv = ['--invalid-option'] - - with self.assertRaises(getopt.GetoptError): - getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - def test_option_combination_username_verbose(self): - """Test combining -u and --verbose options.""" - argv = ['-u', 'testuser', '--verbose'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 2) - self.assertEqual(opts[0], ('-u', 'testuser')) - self.assertEqual(opts[1], ('--verbose', '')) - - def test_option_combination_all_test_flags(self): - """Test combining all test-related flags.""" - argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 4) - opt_dict = dict(opts) - self.assertEqual(opt_dict['-u'], 'testuser') - self.assertIn('--verbose', opt_dict) - self.assertIn('--chat-only', opt_dict) - self.assertIn('--use-chat-downloader-primary', opt_dict) - - def test_option_combination_legacy_mode_with_overrides(self): - """Test legacy mode with multiple overrides.""" - argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 5) - opt_dict = dict(opts) - self.assertIn('--legacy', opt_dict) - self.assertEqual(opt_dict['-q'], '720p') - self.assertEqual(opt_dict['-v'], '1') - self.assertEqual(opt_dict['-c'], '1') - self.assertEqual(opt_dict['-m'], '0') - - -class TestOptionLogicProcessing(unittest.TestCase): - """Test the logic that processes parsed options.""" - - def test_boolean_conversion_true(self): - """Test converting '1' to boolean True.""" - value = '1' - result = bool(int(value)) - self.assertTrue(result) - - def test_boolean_conversion_false(self): - """Test converting '0' to boolean False.""" - value = '0' - result = bool(int(value)) - self.assertFalse(result) - - def test_chat_only_auto_enables_verbose(self): - """Test that chat-only mode should auto-enable verbose.""" - # Simulate the logic from main() - chat_only_mode = True - verbose_mode = False - - if chat_only_mode: - verbose_mode = True - - self.assertTrue(verbose_mode) - - def test_default_chat_downloader_fallback(self): - """Test that chat downloader fallback defaults to enabled.""" - use_chat_downloader_fallback = True # Default - - # Unless explicitly disabled - self.assertTrue(use_chat_downloader_fallback) - - def test_mode_selection_legacy_with_config_json(self): - """Test mode selection logic when config.json exists.""" - # Simulate conditions - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = None - global_config_exists = False - - # Logic from main() - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertTrue(should_use_legacy) - - def test_mode_selection_multi_streamer_with_global_json(self): - """Test mode selection logic when global.json exists.""" - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = None - global_config_exists = True - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertFalse(should_use_legacy) - - def test_mode_selection_multi_streamer_with_username_flag(self): - """Test mode selection when -u flag is used.""" - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = 'testuser' - global_config_exists = False - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertFalse(should_use_legacy) - - def test_mode_selection_explicit_legacy_flag(self): - """Test mode selection with explicit --legacy flag.""" - use_legacy_mode = True - legacy_config_exists = False - specific_streamer = None - global_config_exists = True - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertTrue(should_use_legacy) - - -class TestConfigManager(unittest.TestCase): - """Test ConfigManager functionality.""" - - def setUp(self): - """Set up test fixtures.""" - self.test_dir = tempfile.mkdtemp() - self.original_cwd = os.getcwd() - os.chdir(self.test_dir) - - # Create config directory structure - os.makedirs('config/streamers', exist_ok=True) - - # Patch ConfigManager to use test directory - self.config_dir_patch = patch.object( - ConfigManager, - '__init__', - lambda self: self._init_with_test_dir(self.test_dir) - ) - - def _init_with_test_dir(self, test_dir): - """Initialize ConfigManager with test directory.""" - self.config_dir = Path(test_dir) / "config" - self.streamers_dir = self.config_dir / "streamers" - self.global_config = self._load_global_config() - - def tearDown(self): - """Clean up test fixtures.""" - os.chdir(self.original_cwd) - shutil.rmtree(self.test_dir, ignore_errors=True) - - def test_load_global_config_default(self): - """Test loading default configuration when global.json doesn't exist.""" - # Patch the config_dir to use temp directory - with patch.object(ConfigManager, '_ConfigManager__init__') as mock_init: - manager = ConfigManager.__new__(ConfigManager) - manager.config_dir = Path(self.test_dir) / "config" - manager.streamers_dir = manager.config_dir / "streamers" - manager.global_config = manager._load_global_config() - - # Should have default config values - self.assertIsNotNone(manager.global_config) - self.assertIn('username', manager.global_config) - self.assertIn('quality', manager.global_config) - - def test_load_global_config_from_file(self): - """Test loading global configuration from file.""" - # Create global config - global_config = { - 'quality': '720p', - 'downloadVOD': False, - 'downloadCHAT': True - } - - with open('config/global.json', 'w') as f: - json.dump(global_config, f) - - manager = ConfigManager() - - # Should merge with defaults - self.assertEqual(manager.global_config['quality'], '720p') - self.assertFalse(manager.global_config['downloadVOD']) - self.assertTrue(manager.global_config['downloadCHAT']) - - def test_load_global_config_filters_comments(self): - """Test that global config filters out comment fields.""" - global_config = { - '_comment': 'This is a comment', - 'quality': '720p', - '_note': 'Another comment' - } - - with open('config/global.json', 'w') as f: - json.dump(global_config, f) - - manager = ConfigManager() - - # Comments should be filtered out - self.assertNotIn('_comment', manager.global_config) - self.assertNotIn('_note', manager.global_config) - self.assertEqual(manager.global_config['quality'], '720p') - - def test_load_global_config_filters_schema(self): - """Test that global config filters out $schema field.""" - global_config = { - '$schema': './global.schema.json', - 'quality': '720p' - } - - with open('config/global.json', 'w') as f: - json.dump(global_config, f) - - manager = ConfigManager() - - # $schema should be filtered out - self.assertNotIn('$schema', manager.global_config) - self.assertEqual(manager.global_config['quality'], '720p') - - def test_load_streamer_config_new_streamer(self): - """Test loading config for a new streamer (doesn't exist yet).""" - manager = ConfigManager() - - config = manager.load_streamer_config('newstreamer') - - # Should create default config - self.assertEqual(config['username'], 'newstreamer') - self.assertTrue(config['enabled']) - - # Config file should be created - self.assertTrue(os.path.exists('config/streamers/newstreamer.json')) - - def test_load_streamer_config_existing_streamer(self): - """Test loading config for existing streamer.""" - # Create streamer config - streamer_config = { - 'username': 'existingstreamer', - 'enabled': True, - 'quality': 'source', - 'downloadVOD': True - } - - with open('config/streamers/existingstreamer.json', 'w') as f: - json.dump(streamer_config, f) - - manager = ConfigManager() - config = manager.load_streamer_config('existingstreamer') - - # Should load streamer config - self.assertEqual(config['username'], 'existingstreamer') - self.assertEqual(config['quality'], 'source') - self.assertTrue(config['downloadVOD']) - - def test_load_streamer_config_merges_with_global(self): - """Test that streamer config merges with global config.""" - # Create global config - with open('config/global.json', 'w') as f: - json.dump({ - 'quality': '720p', - 'downloadVOD': True, - 'downloadCHAT': False - }, f) - - # Create streamer config with override - with open('config/streamers/teststreamer.json', 'w') as f: - json.dump({ - 'username': 'teststreamer', - 'enabled': True, - 'quality': 'source' # Override global - }, f) - - manager = ConfigManager() - config = manager.load_streamer_config('teststreamer') - - # Should have streamer's quality (override) - self.assertEqual(config['quality'], 'source') - # Should have global's downloadVOD (inherited) - self.assertTrue(config['downloadVOD']) - # Should have global's downloadCHAT (inherited) - self.assertFalse(config['downloadCHAT']) - - def test_load_streamer_config_filters_comments(self): - """Test that streamer config filters out comments.""" - with open('config/streamers/teststreamer.json', 'w') as f: - json.dump({ - '_comment': 'Test comment', - 'username': 'teststreamer', - 'enabled': True, - '_note': 'Another note' - }, f) - - manager = ConfigManager() - config = manager.load_streamer_config('teststreamer') - - # Comments should be filtered - self.assertNotIn('_comment', config) - self.assertNotIn('_note', config) - self.assertEqual(config['username'], 'teststreamer') - - def test_get_all_enabled_streamers_empty(self): - """Test getting enabled streamers when none exist.""" - manager = ConfigManager() - - streamers = manager.get_all_enabled_streamers() - - self.assertEqual(len(streamers), 0) - - def test_get_all_enabled_streamers_with_enabled(self): - """Test getting enabled streamers.""" - # Create multiple streamer configs - with open('config/streamers/streamer1.json', 'w') as f: - json.dump({'username': 'streamer1', 'enabled': True}, f) - - with open('config/streamers/streamer2.json', 'w') as f: - json.dump({'username': 'streamer2', 'enabled': True}, f) - - with open('config/streamers/streamer3.json', 'w') as f: - json.dump({'username': 'streamer3', 'enabled': False}, f) - - manager = ConfigManager() - streamers = manager.get_all_enabled_streamers() - - # Should return only enabled streamers - self.assertEqual(len(streamers), 2) - self.assertIn('streamer1', streamers) - self.assertIn('streamer2', streamers) - self.assertNotIn('streamer3', streamers) - - def test_get_all_enabled_streamers_default_enabled(self): - """Test that streamers are enabled by default if not specified.""" - # Create config without explicit enabled field - with open('config/streamers/streamer1.json', 'w') as f: - json.dump({'username': 'streamer1'}, f) - - manager = ConfigManager() - streamers = manager.get_all_enabled_streamers() - - # Should be included (defaults to enabled) - self.assertIn('streamer1', streamers) - - def test_create_default_streamer_config(self): - """Test creating default streamer config.""" - manager = ConfigManager() - manager.create_default_streamer_config('newstreamer') - - # File should exist - config_file = Path('config/streamers/newstreamer.json') - self.assertTrue(config_file.exists()) - - # Should have correct structure - with open(config_file) as f: - config = json.load(f) - - self.assertEqual(config['username'], 'newstreamer') - self.assertTrue(config['enabled']) - self.assertIn('$schema', config) - - - - - -if __name__ == '__main__': - # Run tests with verbose output - print("="*70) - print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") - print("="*70) - print() - unittest.main(verbosity=2) diff --git a/test_twitch_archive_simple.py b/test_twitch_archive_simple.py deleted file mode 100644 index 445e741..0000000 --- a/test_twitch_archive_simple.py +++ /dev/null @@ -1,762 +0,0 @@ -""" -Unit tests for Twitch Archive command-line options and configuration. - -Tests focus on: -- Command-line argument parsing (via getopt simulation) -- Options and option combinations -- Configuration logic (filtering, merging, etc.) -- Mode selection logic - -Excludes actual download/processing functionality. - -To run these tests: - python test_twitch_archive_simple.py - or - python -m pytest test_twitch_archive_simple.py -v -""" - -import unittest -import sys -import os -import json -import getopt -import tempfile -import importlib.util -from pathlib import Path -from unittest.mock import patch, MagicMock, Mock - -# Add parent directory to path for imports -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) - -from modules.constants import DEFAULT_CONFIG -from modules.file_manager import FileManager -from modules.downloader import ContentDownloader -from modules.utils import get_ffmpeg_executable, get_twitch_downloader_executable, detect_hardware_acceleration, resolve_hwaccel_type - - -def load_twitch_archive_module(): - """Load the main script module for targeted regression tests.""" - module_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'twitch-archive.py') - spec = importlib.util.spec_from_file_location('twitch_archive_main', module_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -class TestCommandLineArgumentParsing(unittest.TestCase): - """Test command-line argument parsing logic using getopt directly.""" - - def test_help_short_option(self): - """Test -h option parsing.""" - argv = ['-h'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - # Should parse successfully - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0][0], '-h') - - def test_help_long_option(self): - """Test --help option parsing.""" - argv = ['--help'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0][0], '--help') - - def test_username_short_option(self): - """Test -u username option parsing.""" - argv = ['-u', 'teststreamer'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('-u', 'teststreamer')) - - def test_username_long_option(self): - """Test --username option parsing.""" - argv = ['--username', 'teststreamer'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--username', 'teststreamer')) - - def test_verbose_option(self): - """Test --verbose option parsing.""" - argv = ['--verbose'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--verbose', '')) - - def test_chat_only_option(self): - """Test --chat-only option parsing.""" - argv = ['--chat-only'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--chat-only', '')) - - def test_rclone_smoke_test_option(self): - """Test --rclone-smoke-test option parsing.""" - argv = ['--rclone-smoke-test'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--rclone-smoke-test', '')) - - def test_healthcheck_option(self): - """Test --healthcheck option parsing.""" - argv = ['--healthcheck'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--healthcheck', '')) - - def test_legacy_option(self): - """Test --legacy option parsing.""" - argv = ['--legacy'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('--legacy', '')) - - def test_chat_downloader_options(self): - """Test chat downloader option parsing.""" - argv = ['--use-chat-downloader-primary', '--no-chat-downloader-fallback'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 2) - self.assertEqual(opts[0], ('--use-chat-downloader-primary', '')) - self.assertEqual(opts[1], ('--no-chat-downloader-fallback', '')) - - def test_legacy_quality_option(self): - """Test -q quality option parsing.""" - argv = ['-q', '720p'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 1) - self.assertEqual(opts[0], ('-q', '720p')) - - def test_legacy_boolean_options(self): - """Test legacy boolean option parsing.""" - argv = ['-v', '1', '-c', '0', '-m', '1'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 3) - self.assertEqual(opts[0], ('-v', '1')) - self.assertEqual(opts[1], ('-c', '0')) - self.assertEqual(opts[2], ('-m', '1')) - - def test_invalid_option(self): - """Test that invalid option raises error.""" - argv = ['--invalid-option'] - - with self.assertRaises(getopt.GetoptError): - getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - def test_option_combination_username_verbose(self): - """Test combining -u and --verbose options.""" - argv = ['-u', 'testuser', '--verbose'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 2) - self.assertEqual(opts[0], ('-u', 'testuser')) - self.assertEqual(opts[1], ('--verbose', '')) - - def test_option_combination_all_test_flags(self): - """Test combining all test-related flags.""" - argv = ['-u', 'testuser', '--verbose', '--chat-only', '--use-chat-downloader-primary'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 4) - opt_dict = dict(opts) - self.assertEqual(opt_dict['-u'], 'testuser') - self.assertIn('--verbose', opt_dict) - self.assertIn('--chat-only', opt_dict) - self.assertIn('--use-chat-downloader-primary', opt_dict) - - def test_option_combination_legacy_mode_with_overrides(self): - """Test legacy mode with multiple overrides.""" - argv = ['--legacy', '-q', '720p', '-v', '1', '-c', '1', '-m', '0'] - opts, args = getopt.getopt( - argv, - "hu:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - - self.assertEqual(len(opts), 5) - opt_dict = dict(opts) - self.assertIn('--legacy', opt_dict) - self.assertEqual(opt_dict['-q'], '720p') - self.assertEqual(opt_dict['-v'], '1') - self.assertEqual(opt_dict['-c'], '1') - self.assertEqual(opt_dict['-m'], '0') - - -class TestOptionLogicProcessing(unittest.TestCase): - """Test the logic that processes parsed options.""" - - def test_boolean_conversion_true(self): - """Test converting '1' to boolean True.""" - value = '1' - result = bool(int(value)) - self.assertTrue(result) - - def test_boolean_conversion_false(self): - """Test converting '0' to boolean False.""" - value = '0' - result = bool(int(value)) - self.assertFalse(result) - - def test_chat_only_auto_enables_verbose(self): - """Test that chat-only mode should auto-enable verbose.""" - # Simulate the logic from main() - chat_only_mode = True - verbose_mode = False - - if chat_only_mode: - verbose_mode = True - - self.assertTrue(verbose_mode) - - def test_default_chat_downloader_fallback(self): - """Test that chat downloader fallback defaults to enabled.""" - use_chat_downloader_fallback = True # Default value - - # Unless explicitly disabled - self.assertTrue(use_chat_downloader_fallback) - - def test_mode_selection_legacy_with_config_json(self): - """Test mode selection logic when config.json exists.""" - # Simulate conditions - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = None - global_config_exists = False - - # Logic from main() function - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertTrue(should_use_legacy) - - def test_mode_selection_multi_streamer_with_global_json(self): - """Test mode selection logic when global.json exists.""" - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = None - global_config_exists = True - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertFalse(should_use_legacy) - - def test_mode_selection_multi_streamer_with_username_flag(self): - """Test mode selection when -u flag is used.""" - use_legacy_mode = False - legacy_config_exists = True - specific_streamer = 'testuser' - global_config_exists = False - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertFalse(should_use_legacy) - - def test_mode_selection_explicit_legacy_flag(self): - """Test mode selection with explicit --legacy flag.""" - use_legacy_mode = True - legacy_config_exists = False - specific_streamer = None - global_config_exists = True - - should_use_legacy = use_legacy_mode or ( - legacy_config_exists and not specific_streamer and not global_config_exists - ) - - self.assertTrue(should_use_legacy) - - -class TestConfigLogic(unittest.TestCase): - """Test configuration management logic.""" - - def test_comment_filtering_logic(self): - """Test that comment fields are filtered out.""" - user_config = { - '_comment': 'This is a comment', - 'quality': '720p', - '_note': 'Another comment', - 'username': 'testuser' - } - - # Apply the filtering logic (same as in ConfigManager) - filtered = {k: v for k, v in user_config.items() if not k.startswith('_')} - - self.assertNotIn('_comment', filtered) - self.assertNotIn('_note', filtered) - self.assertIn('quality', filtered) - self.assertIn('username', filtered) - self.assertEqual(filtered['quality'], '720p') - - def test_schema_filtering_logic(self): - """Test that $schema field is filtered out.""" - user_config = { - '$schema': './config.schema.json', - 'quality': '720p', - 'username': 'testuser' - } - - # Apply the filtering logic (same as in ConfigManager) - filtered = {k: v for k, v in user_config.items() - if not k.startswith('_') and k != '$schema'} - - self.assertNotIn('$schema', filtered) - self.assertIn('quality', filtered) - self.assertIn('username', filtered) - - def test_config_merging_logic(self): - """Test config merging logic (streamer overrides global).""" - global_config = { - 'quality': '720p', - 'downloadVOD': True, - 'downloadCHAT': False, - 'username': 'default' - } - - streamer_config = { - 'username': 'specificstreamer', - 'quality': 'source', # Override - # downloadVOD and downloadCHAT inherited from global - } - - # Simulate merging (same as in ConfigManager.load_streamer_config) - merged = global_config.copy() - merged.update(streamer_config) - - # Check overrides - self.assertEqual(merged['quality'], 'source') # Overridden - self.assertEqual(merged['username'], 'specificstreamer') # Overridden - # Check inherited values - self.assertTrue(merged['downloadVOD']) - self.assertFalse(merged['downloadCHAT']) - - def test_default_config_structure(self): - """Test that DEFAULT_CONFIG has expected keys.""" - self.assertIn('username', DEFAULT_CONFIG) - self.assertIn('quality', DEFAULT_CONFIG) - self.assertIn('downloadVOD', DEFAULT_CONFIG) - self.assertIn('downloadCHAT', DEFAULT_CONFIG) - self.assertIn('downloadMETADATA', DEFAULT_CONFIG) - self.assertIn('uploadCloud', DEFAULT_CONFIG) - self.assertIn('deleteFiles', DEFAULT_CONFIG) - self.assertIn('notifications', DEFAULT_CONFIG) - self.assertIn('refresh', DEFAULT_CONFIG) - - def test_enabled_flag_filtering_logic(self): - """Test filtering streamers by enabled flag.""" - streamers = [ - {'username': 'streamer1', 'enabled': True}, - {'username': 'streamer2', 'enabled': True}, - {'username': 'streamer3', 'enabled': False}, - {'username': 'streamer4'}, # No enabled field - ] - - # Simulate filtering logic (same as in ConfigManager.get_all_enabled_streamers) - enabled_streamers = [ - s['username'] for s in streamers - if s.get('enabled', True) # Default to True if not specified - ] - - self.assertIn('streamer1', enabled_streamers) - self.assertIn('streamer2', enabled_streamers) - self.assertNotIn('streamer3', enabled_streamers) - self.assertIn('streamer4', enabled_streamers) # Default enabled - - def test_default_streamer_config_structure(self): - """Test default streamer config structure.""" - # Simulate what create_default_streamer_config creates - username = 'newstreamer' - default_config = { - "$schema": "../streamer.schema.json", - "username": username, - "enabled": True - } - - self.assertEqual(default_config['username'], username) - self.assertTrue(default_config['enabled']) - self.assertIn('$schema', default_config) - - -class TestFileManagerUploadPaths(unittest.TestCase): - """Test rclone upload path preparation.""" - - def test_build_upload_relative_paths_uses_forward_slashes(self): - """Rclone --files-from entries must use POSIX separators on Windows.""" - with tempfile.TemporaryDirectory() as temp_dir: - manager = FileManager( - root_path=temp_dir, - username='testuser', - config={ - 'uploadCloud': True, - 'uploadPreMergeVideo': True, - 'uploadMergedVideo': True, - 'uploadChatVideo': True - } - ) - - relative_paths = manager._build_upload_relative_paths('20260424_12h00m00s') - - self.assertTrue(relative_paths) - self.assertTrue(all('\\' not in path for path in relative_paths)) - self.assertIn('testuser/metadata/METADA_20260424_12h00m00s.json', relative_paths) - self.assertIn('testuser/chat/json/CHAT_20260424_12h00m00s.json', relative_paths) - - -class TestDownloaderConfiguration(unittest.TestCase): - """Regression tests for downloader config wiring.""" - - def test_download_vod_method_not_shadowed_by_boolean_flag(self): - """Config booleans must not overwrite callable downloader methods.""" - downloader = ContentDownloader( - twitch_downloader_path='TwitchDownloaderCLI', - ffmpeg_path='ffmpeg', - config={ - 'downloadVOD': True, - 'downloadCHAT': True, - 'downloadLiveCHAT': True - } - ) - - self.assertTrue(callable(downloader.download_vod)) - self.assertTrue(downloader.download_vod_enabled) - - -class TestLinuxToolResolution(unittest.TestCase): - """Ensure Linux containers prefer their own installed toolchain.""" - - @patch('modules.utils.shutil.which') - def test_linux_prefers_system_ffmpeg(self, mock_which): - mock_which.return_value = '/usr/bin/ffmpeg' - - self.assertEqual(get_ffmpeg_executable('linux'), '/usr/bin/ffmpeg') - - @patch('modules.utils.shutil.which') - def test_linux_prefers_system_twitch_downloader(self, mock_which): - mock_which.return_value = '/usr/local/bin/TwitchDownloaderCLI' - - self.assertEqual(get_twitch_downloader_executable('linux'), '/usr/local/bin/TwitchDownloaderCLI') - - @patch('modules.utils.os.path.exists', return_value=False) - @patch('modules.utils.shutil.which', return_value=None) - def test_linux_auto_hwaccel_falls_back_to_software_without_runtime(self, _mock_which, _mock_exists): - detected = detect_hardware_acceleration('auto', 'linux') - - self.assertEqual(detected, 'none') - self.assertEqual(resolve_hwaccel_type(detected, 'linux'), 'none') - - @patch('modules.utils.shutil.which', return_value='/usr/bin/nvidia-smi') - def test_linux_auto_hwaccel_uses_nvenc_when_nvidia_runtime_visible(self, _mock_which): - detected = detect_hardware_acceleration('auto', 'linux') - - self.assertEqual(detected, 'nvenc') - - -class TestChatRenderBehavior(unittest.TestCase): - """Regression tests for chat rendering defaults and retry behavior.""" - - @patch('modules.downloader.sys.platform', 'linux') - @patch('modules.downloader.subprocess.Popen') - def test_linux_chat_render_uses_container_safe_font(self, mock_popen): - with tempfile.TemporaryDirectory() as temp_dir: - json_path = os.path.join(temp_dir, 'chat.json') - video_path = os.path.join(temp_dir, 'chat.mp4') - - with open(json_path, 'w', encoding='utf-8') as handle: - json.dump( - { - 'comments': [ - { - 'message': {'body': 'hello world ' * 20}, - 'commenter': {'display_name': 'tester'} - } - ] - }, - handle - ) - - class FakeProcess: - def __init__(self, output_path: str): - self.stdout = iter(['rendering']) - self.output_path = output_path - - def wait(self): - with open(self.output_path, 'wb') as handle: - handle.write(b'x' * 2048) - return 0 - - captured = {} - - def build_process(cmd, **_kwargs): - captured['cmd'] = cmd - return FakeProcess(video_path) - - mock_popen.side_effect = build_process - - downloader = ContentDownloader('TwitchDownloaderCLI', 'ffmpeg', {'downloadCHAT': True}) - - result = downloader.render_chat(json_path, video_path, '-c:v libx264 "{save_path}"') - - self.assertTrue(result) - self.assertIn('DejaVu Sans', captured['cmd']) - - -class TestMultiStreamerCleanupRegression(unittest.TestCase): - """Regression tests for multi-streamer conversion and cleanup behavior.""" - - def setUp(self): - self.module = load_twitch_archive_module() - - def _build_archiver(self, temp_dir: str, upload_cloud: bool = True): - archiver = MagicMock() - archiver.username = 'maddoscientist0' - archiver.os_type = 'linux' - archiver.quality = 'best' - archiver.downloadLiveCHAT = False - archiver.downloadCHAT = False - archiver.downloadVOD = False - archiver.downloadMETADATA = False - archiver.mergeVideoChat = False - archiver.mergeChatLayout = 'side-by-side' - archiver.onlyRaw = False - archiver.vodTimeout = 0 - archiver.shutdown_requested = False - archiver.deleteFiles = True - archiver.uploadCloud = upload_cloud - - archiver.notification_manager = MagicMock() - archiver.recorder = MagicMock() - archiver.processor = MagicMock() - archiver.downloader = MagicMock() - archiver.stream_monitor = MagicMock() - archiver.file_manager = MagicMock() - archiver.file_manager.raw_path = Path(temp_dir) / 'raw' - archiver.file_manager.video_path = Path(temp_dir) / 'video' - archiver.file_manager.chat_json_path = Path(temp_dir) / 'chat_json' - archiver.file_manager.chat_mp4_path = Path(temp_dir) / 'chat' - - os.makedirs(archiver.file_manager.raw_path, exist_ok=True) - os.makedirs(archiver.file_manager.video_path, exist_ok=True) - os.makedirs(archiver.file_manager.chat_json_path, exist_ok=True) - os.makedirs(archiver.file_manager.chat_mp4_path, exist_ok=True) - - return archiver - - def test_process_stream_keeps_raw_when_conversion_fails(self): - manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') - - with tempfile.TemporaryDirectory() as temp_dir: - archiver = self._build_archiver(temp_dir) - archiver.processor.process_raw_stream.return_value = False - archiver.file_manager.upload_to_cloud.return_value = True - - def write_raw_file(_stream_info, raw_path): - with open(raw_path, 'wb') as handle: - handle.write(b'x' * 4096) - return True - - archiver.recorder.record.side_effect = write_raw_file - - stream_info = { - 'title': 'Test', - 'createdAt': '2026-04-25T09:14:01Z' - } - - manager._process_stream(archiver, stream_info, 'stream-id') - - archiver.file_manager.clean_raw_file.assert_not_called() - archiver.file_manager.delete_local_files.assert_called_once() - - def test_process_stream_only_deletes_rendered_files_after_real_upload(self): - manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') - - with tempfile.TemporaryDirectory() as temp_dir: - archiver = self._build_archiver(temp_dir, upload_cloud=False) - archiver.processor.process_raw_stream.return_value = True - archiver.file_manager.upload_to_cloud.return_value = True - - def write_raw_file(_stream_info, raw_path): - with open(raw_path, 'wb') as handle: - handle.write(b'x' * 4096) - return True - - archiver.recorder.record.side_effect = write_raw_file - - stream_info = { - 'title': 'Test', - 'createdAt': '2026-04-25T09:14:01Z' - } - - manager._process_stream(archiver, stream_info, 'stream-id') - - archiver.file_manager.clean_raw_file.assert_called_once() - archiver.file_manager.delete_local_files.assert_not_called() - - upload_filename_base = archiver.file_manager.upload_to_cloud.call_args.args[0] - self.assertFalse(upload_filename_base.startswith('LIVE_')) - - def test_process_stream_preserves_files_when_chat_render_fails(self): - manager = self.module.TwitchArchiveManager(specific_streamer='maddoscientist0') - - with tempfile.TemporaryDirectory() as temp_dir: - archiver = self._build_archiver(temp_dir) - archiver.downloadCHAT = True - archiver.downloadVOD = True - archiver.vodTimeout = 30 - archiver.processor.process_raw_stream.return_value = True - archiver.file_manager.upload_to_cloud.return_value = True - archiver.downloader.last_chat_render_attempted = False - archiver.downloader.last_chat_render_succeeded = False - - def write_raw_file(_stream_info, raw_path): - with open(raw_path, 'wb') as handle: - handle.write(b'x' * 4096) - return True - - def failed_chat_render(*_args, **_kwargs): - archiver.downloader.last_chat_render_attempted = True - archiver.downloader.last_chat_render_succeeded = False - return False - - archiver.recorder.record.side_effect = write_raw_file - archiver.downloader.download_and_render_chat.side_effect = failed_chat_render - archiver.stream_monitor.get_latest_vod.return_value = { - 'data': { - 'user': { - 'videos': { - 'edges': [ - { - 'node': { - 'id': '2756589076', - 'title': 'Test', - 'recordedAt': '2026-04-25T09:14:01Z' - } - } - ] - } - } - } - } - - stream_info = { - 'title': 'Test', - 'createdAt': '2026-04-25T09:14:01Z' - } - - manager._process_stream(archiver, stream_info, 'stream-id') - - archiver.file_manager.clean_raw_file.assert_not_called() - archiver.file_manager.delete_local_files.assert_not_called() - - -if __name__ == '__main__': - # Run tests with verbose output - print("="*70) - print("TWITCH ARCHIVE - Unit Tests for Options and Configuration") - print("="*70) - print() - unittest.main(verbosity=2) diff --git a/twitch-archive.py b/twitch-archive.py index 707bdfe..5844cde 100644 --- a/twitch-archive.py +++ b/twitch-archive.py @@ -34,8 +34,6 @@ import time import json import signal import getopt -import pathlib -import subprocess from typing import Dict, Optional, Any from datetime import datetime, timedelta @@ -50,8 +48,7 @@ from modules.config import ConfigManager from modules.notifications import NotificationManager from modules.utils import ( detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader, - verify_rclone, get_env_value + get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader ) from modules.stream_monitor import StreamMonitor from modules.recorder import StreamRecorder @@ -150,25 +147,10 @@ class TwitchArchive: Raises: SystemExit: If .env file is not found """ - dotenv_loaded = load_dotenv(find_dotenv()) - has_required_env = bool( - get_env_value('CLIENT-ID', 'CLIENT_ID') and - get_env_value('CLIENT-SECRET', 'CLIENT_SECRET') - ) - - if not dotenv_loaded and has_required_env: - print(f'{Fore.GREEN}✓ Twitch API credentials loaded from process environment{Style.RESET_ALL}') - return - - if not dotenv_loaded and not has_required_env: + if not load_dotenv(find_dotenv()): print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials or pass them via environment variables{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') - sys.exit(1) - - if not has_required_env: - print(f'{Fore.RED}✗ ERROR: Twitch API credentials are missing{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID/CLIENT_ID and CLIENT-SECRET/CLIENT_SECRET{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') + print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') sys.exit(1) def _initialize_components(self) -> None: @@ -277,8 +259,6 @@ class TwitchArchive: verify_ffmpeg(self.os_type) if self.downloadVOD or self.downloadCHAT: verify_twitch_downloader(self.os_type) - if self.uploadCloud and not verify_rclone(): - sys.exit(1) # Print configuration summary self._print_configuration_summary() @@ -424,8 +404,7 @@ class TwitchArchive: print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') # Process the raw stream file - processing_succeeded = self.processor.process_raw_stream(live_raw_path, live_proc_path) - self.downloader.reset_chat_render_status() + self.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -447,11 +426,7 @@ class TwitchArchive: else: # Get video duration first (needed for chat conversion and trimming) ffmpeg_path = get_ffmpeg_executable(self.os_type) - if not processing_succeeded or not os.path.exists(live_proc_path): - print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') - video_duration = None - else: - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') # Convert chat format if needed (chat_downloader uses different JSON structure) @@ -586,17 +561,7 @@ class TwitchArchive: print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') # Clean up raw files if configured - chat_render_retry_needed = ( - self.downloader.last_chat_render_attempted and - not self.downloader.last_chat_render_succeeded - ) - - if chat_render_retry_needed: - print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') - elif processing_succeeded: - self.file_manager.clean_raw_file(live_raw_path) - elif os.path.exists(live_raw_path): - print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') + self.file_manager.clean_raw_file(live_raw_path) # Upload to cloud if configured upload_success = self.file_manager.upload_to_cloud( @@ -605,7 +570,7 @@ class TwitchArchive: ) # Delete local files if configured and upload succeeded - if self.deleteFiles and self.uploadCloud and upload_success and not chat_render_retry_needed: + if self.deleteFiles and upload_success: self.file_manager.delete_local_files( filename_base, live_raw_path, @@ -925,8 +890,6 @@ class TwitchArchiveManager: verify_ffmpeg(first_archiver.os_type) if first_archiver.downloadVOD or first_archiver.downloadCHAT: verify_twitch_downloader(first_archiver.os_type) - if any(archiver.uploadCloud for archiver in self.archivers.values()) and not verify_rclone(): - sys.exit(1) # Print configuration summary for each streamer for username, archiver in self.archivers.items(): @@ -1055,7 +1018,7 @@ class TwitchArchiveManager: # Generate timestamp and filename timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{archiver.username}_{timestamp}" + filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" # Parse stream start time live_date = datetime.strptime( @@ -1066,8 +1029,8 @@ class TwitchArchiveManager: raw_extension = '.ts' proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - live_raw_path = str(archiver.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{proc_extension}") + live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") + live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") # Send notification @@ -1125,7 +1088,6 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, verbose=self.verbose ) except Exception as e: @@ -1154,7 +1116,6 @@ class TwitchArchiveManager: archiver.downloader.start_chat_downloader_thread( archiver.username, chat_json_path, shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, verbose=self.verbose or self.chat_only ) # Wait for completion @@ -1209,10 +1170,8 @@ class TwitchArchiveManager: print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') # Process raw stream - processing_succeeded = False if not archiver.onlyRaw: - processing_succeeded = archiver.processor.process_raw_stream(live_raw_path, live_proc_path) - archiver.downloader.reset_chat_render_status() + archiver.processor.process_raw_stream(live_raw_path, live_proc_path) # Wait for live chat download if it was started live_chat_downloaded = False @@ -1251,12 +1210,8 @@ class TwitchArchiveManager: chat_rendered_successfully = False else: # Get video duration first - if not processing_succeeded or not os.path.exists(live_proc_path): - print(f'{Fore.YELLOW}⚠ Processed video file is unavailable, skipping chat render{Style.RESET_ALL}') - video_duration = None - else: - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) + ffmpeg_path = get_ffmpeg_executable(archiver.os_type) + video_duration = get_video_duration(live_proc_path, ffmpeg_path) if video_duration is None: print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') @@ -1405,17 +1360,7 @@ class TwitchArchiveManager: archiver.file_manager.save_metadata(stream_info, filename_base) # Clean up raw file if configured - chat_render_retry_needed = ( - archiver.downloader.last_chat_render_attempted and - not archiver.downloader.last_chat_render_succeeded - ) - - if chat_render_retry_needed: - print(f'{Fore.YELLOW}⚠ Preserving local files because chat rendering failed and can be retried later{Style.RESET_ALL}') - elif processing_succeeded: - archiver.file_manager.clean_raw_file(live_raw_path) - elif os.path.exists(live_raw_path): - print(f'{Fore.YELLOW}⚠ Keeping raw file because conversion did not complete successfully{Style.RESET_ALL}') + archiver.file_manager.clean_raw_file(live_raw_path) # Upload to cloud if configured upload_success = archiver.file_manager.upload_to_cloud( @@ -1424,7 +1369,7 @@ class TwitchArchiveManager: ) # Delete files if configured - if archiver.deleteFiles and archiver.uploadCloud and upload_success and not chat_render_retry_needed: + if archiver.deleteFiles and upload_success: archiver.file_manager.delete_local_files( filename_base, live_raw_path, @@ -1439,79 +1384,6 @@ class TwitchArchiveManager: ) -def run_rclone_smoke_test(specific_streamer: Optional[str] = None) -> int: - """Run a one-off rclone smoke test using the configured upload destination.""" - config_manager = ConfigManager() - - if specific_streamer: - username = specific_streamer - else: - enabled_streamers = config_manager.get_all_enabled_streamers() - if not enabled_streamers: - print(f'{Fore.RED}✗ No enabled streamers available for smoke test{Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Use -u or enable a streamer config{Style.RESET_ALL}') - return 1 - username = enabled_streamers[0] - - config = config_manager.load_streamer_config(username) - file_manager = FileManager( - root_path=config.get('root_path', 'archive'), - username=username, - config=config - ) - file_manager.initialize_directories() - - print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') - print(f'{Fore.CYAN}TWITCH ARCHIVE - Rclone Smoke Test{Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') - print(f'{Fore.GREEN}Streamer: {username}{Style.RESET_ALL}') - print(f'{Fore.GREEN}Remote: {config.get("rclone_path", "")}{Style.RESET_ALL}\n') - - return 0 if file_manager.run_rclone_smoke_test() else 1 - - -def run_healthcheck(specific_streamer: Optional[str] = None) -> int: - """Run a local readiness check suitable for Docker health checks.""" - config_manager = ConfigManager() - - if specific_streamer: - username = specific_streamer - else: - enabled_streamers = config_manager.get_all_enabled_streamers() - username = enabled_streamers[0] if enabled_streamers else 'vinesauce' - - config = config_manager.load_streamer_config(username) - archive = TwitchArchive(config) - - try: - archive._load_environment_variables() - except SystemExit: - return 1 - - archive._initialize_components() - - checks_ok = True - if not verify_streamlink(): - checks_ok = False - if not verify_ffmpeg(archive.os_type): - checks_ok = False - if (archive.downloadVOD or archive.downloadCHAT) and not verify_twitch_downloader(archive.os_type): - checks_ok = False - if archive.uploadCloud: - if not verify_rclone(): - checks_ok = False - rclone_config_path = os.getenv('RCLONE_CONFIG') - if rclone_config_path and not os.path.exists(rclone_config_path): - print(f'{Fore.RED}✗ ERROR: RCLONE_CONFIG points to a missing file: {rclone_config_path}{Style.RESET_ALL}') - checks_ok = False - - if not checks_ok: - return 1 - - print(f'{Fore.GREEN}✓ Healthcheck OK for {username}{Style.RESET_ALL}') - return 0 - - # ============================================================================ # COMMAND-LINE INTERFACE # ============================================================================ @@ -1527,8 +1399,6 @@ def main(argv: list) -> None: """ specific_streamer = None use_legacy_mode = False - rclone_smoke_test_mode = False - healthcheck_mode = False help_msg = f''' {Fore.CYAN}{"=" * 70} @@ -1555,8 +1425,6 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving --legacy Force legacy mode (use config.json) --chat-only Test mode: Only download chat (skip video recording) Automatically enables verbose logging - --healthcheck Validate config and tool availability, then exit - --rclone-smoke-test Create a small test file and upload it with rclone --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) --no-chat-downloader-fallback Disable chat_downloader fallback @@ -1594,7 +1462,7 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving "h:u:q:a:v:c:m:r:d:n:", ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "healthcheck", "rclone-smoke-test", "use-chat-downloader-primary", "no-chat-downloader-fallback"] + "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] ) except getopt.GetoptError as e: print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') @@ -1621,10 +1489,6 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving elif opt == "--chat-only": chat_only_mode = True verbose_mode = True # Auto-enable verbose for chat-only mode - elif opt == "--healthcheck": - healthcheck_mode = True - elif opt == "--rclone-smoke-test": - rclone_smoke_test_mode = True elif opt == "--legacy": use_legacy_mode = True elif opt == "--use-chat-downloader-primary": @@ -1657,12 +1521,6 @@ TWITCH ARCHIVE - Automated Stream Recording & Archiving legacy_overrides['deleteFiles'] = bool(int(arg)) elif opt in ("-n", "--notifications"): legacy_overrides['notifications'] = bool(int(arg)) - - if rclone_smoke_test_mode: - sys.exit(run_rclone_smoke_test(specific_streamer)) - - if healthcheck_mode: - sys.exit(run_healthcheck(specific_streamer)) # Determine which mode to use if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')):