diff --git a/.github/instructions/blazor.instructions.md b/.github/instructions/blazor.instructions.md new file mode 100644 index 0000000..4e88cc0 --- /dev/null +++ b/.github/instructions/blazor.instructions.md @@ -0,0 +1,77 @@ +--- +description: 'Blazor component and application patterns' +applyTo: '**/*.razor, **/*.razor.cs, **/*.razor.css' +--- + +## Blazor Code Style and Structure + +- Write idiomatic and efficient Blazor and C# code. +- Follow .NET and Blazor conventions. +- Use Razor Components appropriately for component-based UI development. +- Prefer inline functions for smaller components but separate complex logic into code-behind or service classes. +- Async/await should be used where applicable to ensure non-blocking UI operations. + +## Naming Conventions + +- Follow PascalCase for component names, method names, and public members. +- Use camelCase for private fields and local variables. +- Prefix interface names with "I" (e.g., IUserService). + +## Blazor and .NET Specific Guidelines + +- Utilize Blazor's built-in features for component lifecycle (e.g., OnInitializedAsync, OnParametersSetAsync). +- Use data binding effectively with @bind. +- Leverage Dependency Injection for services in Blazor. +- Structure Blazor components and services following Separation of Concerns. +- Always use the latest version C#, currently C# 13 features like record types, pattern matching, and global usings. + +## Error Handling and Validation + +- Implement proper error handling for Blazor pages and API calls. +- Use logging for error tracking in the backend and consider capturing UI-level errors in Blazor with tools like ErrorBoundary. +- Implement validation using FluentValidation or DataAnnotations in forms. + +## Blazor API and Performance Optimization + +- Utilize Blazor server-side or WebAssembly optimally based on the project requirements. +- Use asynchronous methods (async/await) for API calls or UI actions that could block the main thread. +- Optimize Razor components by reducing unnecessary renders and using StateHasChanged() efficiently. +- Minimize the component render tree by avoiding re-renders unless necessary, using ShouldRender() where appropriate. +- Use EventCallbacks for handling user interactions efficiently, passing only minimal data when triggering events. + +## Caching Strategies + +- Implement in-memory caching for frequently used data, especially for Blazor Server apps. Use IMemoryCache for lightweight caching solutions. +- For Blazor WebAssembly, utilize localStorage or sessionStorage to cache application state between user sessions. +- Consider Distributed Cache strategies (like Redis or SQL Server Cache) for larger applications that need shared state across multiple users or clients. +- Cache API calls by storing responses to avoid redundant calls when data is unlikely to change, thus improving the user experience. + +## State Management Libraries + +- Use Blazor's built-in Cascading Parameters and EventCallbacks for basic state sharing across components. +- Implement advanced state management solutions using libraries like Fluxor or BlazorState when the application grows in complexity. +- For client-side state persistence in Blazor WebAssembly, consider using Blazored.LocalStorage or Blazored.SessionStorage to maintain state between page reloads. +- For server-side Blazor, use Scoped Services and the StateContainer pattern to manage state within user sessions while minimizing re-renders. + +## API Design and Integration + +- Use HttpClient or other appropriate services to communicate with external APIs or your own backend. +- Implement error handling for API calls using try-catch and provide proper user feedback in the UI. + +## Testing and Debugging in Visual Studio + +- All unit testing and integration testing should be done in Visual Studio. +- Test Blazor components and services using xUnit, NUnit, or MSTest. +- Use Moq or NSubstitute for mocking dependencies during tests. +- Debug Blazor UI issues using browser developer tools and Visual Studio's debugging tools for backend and server-side issues. +- For performance profiling and optimization, rely on Visual Studio's diagnostics tools. + +## Security and Authentication + +- Implement Authentication and Authorization in the Blazor app where necessary using ASP.NET Identity or JWT tokens for API authentication. +- Use HTTPS for all web communication and ensure proper CORS policies are implemented. + +## API Documentation and Swagger + +- Use Swagger/OpenAPI for API documentation for your backend API services. +- Ensure XML documentation for models and API methods for enhancing Swagger documentation. diff --git a/.gitignore b/.gitignore index b1a3beb..7433967 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,48 @@ venv3/** .gitignore bin/** \n+# Ignore any virtual environment directories starting with 'venv' (venv, venv3, venv314, etc.) -venv*/ \ No newline at end of file +venv*/ +.vs/ProjectEvaluation/twitch-archive-2.metadata.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.projects.v10.bin +.vs/ProjectEvaluation/twitch-archive-2.strings.v10.bin +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/CodeChunks.db +.vs/Twitch-Archive-2/CopilotIndices/18.3.508.13148/SemanticSymbols.db +.vs/Twitch-Archive-2/DesignTimeBuild/.dtbcache.v2 +.vs/Twitch-Archive-2/FileContentIndex/843065c8-d80f-4907-b0ae-6d010b3a5699.vsidx +.vs/Twitch-Archive-2/FileContentIndex/ef7e1a3c-80cd-4867-a9a8-2e5099471227.vsidx +.vs/Twitch-Archive-2/v18/.futdcache.v2 +.vs/Twitch-Archive-2/v18/.suo +.vs/Twitch-Archive-2/v18/DocumentLayout.backup.json +.vs/Twitch-Archive-2/v18/DocumentLayout.json +.vscode/settings.json + +# C# / Visual Studio +# Build Folders +bin/ +obj/ + +# Visual Studio files +*.user +*.suo +*.userprefs +*.csproj.user +*.pidb +*.pdb +*.cache +*.ilk +*.log +*.vspscc +*.vssscc + +# Test results and packages +TestResults/ +packages/ +*.nupkg + +# Database and backup +*.dbmdl +*.bak +*.backup +*.orig + +dotnet/.vs/** \ No newline at end of file diff --git a/Twitch-Archive-2.sln b/Twitch-Archive-2.sln new file mode 100644 index 0000000..a41b4f2 --- /dev/null +++ b/Twitch-Archive-2.sln @@ -0,0 +1,32 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dotnet", "dotnet", "{71E6E750-85FD-B5BC-4321-E01377EC6231}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D90AB541-7400-80B1-A0B4-F58D0D439F55}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TwitchArchive.Core", "dotnet\src\TwitchArchive.Core\TwitchArchive.Core.csproj", "{1D11D744-6D0D-BB4D-8B77-30B5CE764821}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1D11D744-6D0D-BB4D-8B77-30B5CE764821}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {D90AB541-7400-80B1-A0B4-F58D0D439F55} = {71E6E750-85FD-B5BC-4321-E01377EC6231} + {1D11D744-6D0D-BB4D-8B77-30B5CE764821} = {D90AB541-7400-80B1-A0B4-F58D0D439F55} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {D101688C-0CA3-4CFB-96D4-E1AB9A62EC51} + EndGlobalSection +EndGlobal diff --git a/UpgradePlan.md b/UpgradePlan.md new file mode 100644 index 0000000..6654d3d --- /dev/null +++ b/UpgradePlan.md @@ -0,0 +1,214 @@ +Plan: C# .NET 10 Twitch Archive Rewrite +A complete port of the Python archiver to C# .NET 10 with Blazor Server UI, real-time process output, SQLite state tracking, full DI/service pattern, NLog logging, and a resilient recording engine. Placed under dotnet/ in the existing repo. + +Project Layout +Step 1 — Solution & Project scaffolding +Create the SLN and three projects: + +TwitchArchive.Core — classlib, targets net10.0 +TwitchArchive.Web — Blazor Server (blazorserver), targets net10.0 +TwitchArchive.Tests — xUnit, targets net10.0 +NuGet packages: + +Core: Microsoft.EntityFrameworkCore.Sqlite, Polly, NLog, NLog.Extensions.Logging +Web: all Core packages + Microsoft.AspNetCore.SignalR, NLog.Web.AspNetCore +Tests: xunit, Moq, coverlet.collector, Microsoft.EntityFrameworkCore.InMemory +Step 2 — Configuration models +Mirror the existing JSON schemas as C# POCOs with System.Text.Json attributes: + +GlobalConfig.cs — one property per key in config/global.json.example +StreamerConfig.cs — all fields nullable (override semantics), only Username and Enabled required +EffectiveConfig.cs — computed merge of global + per-streamer; exposes resolved values +AppSettings.cs — app-level settings (password hash, tool paths, .env secrets) +IConfigurationService / ConfigurationService: + +LoadGlobal() / SaveGlobal(GlobalConfig) +LoadStreamer(string username) / SaveStreamer(StreamerConfig) +GetAllStreamers(), GetEffectiveConfig(string username) (merge logic) +Reads/writes global.json and config/streamers/*.json — same files as Python +Step 3 — Infrastructure layer +TwitchApiClient (injectable, mockable): + +GetOAuthTokenAsync() — POST to https://id.twitch.tv/oauth2/token, caches token, refreshes on 401 +CheckStreamStatusAsync(string username) — GQL query for live stream + archiveVideo.id +GetLatestVodAsync(string username) — GQL query for most recent VOD +ValidateUsernameAsync(string username) — Helix /users endpoint +Credentials read from environment (CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN) +All methods return typed result objects, never throw on network errors — return Result (or OneOf) +HttpResiliencePolicy (Polly): + +Wraps HttpClient for TwitchApiClient +WaitAndRetryForever with exponential backoff starting at 15 s, doubling, capped at 10 minutes +Only applies to transient errors (5xx, timeout, HttpRequestException) — not 401/404 +Logged via NLog on each retry attempt +ProcessRunner (injectable + mockable for tests): + +RunAsync(ProcessRunOptions options, CancellationToken ct) → int exitCode +StartAsync(ProcessRunOptions options, CancellationToken ct) → IRunningProcess handle (for long-lived processes like streamlink) +Reads stdout and stderr line by line asynchronously +Reports each line to IProcessOutputStore (streamer + job context) +Forwards to NLog +ProcessRunOptions: FileName, Arguments, WorkingDirectory, RedirectOutput + +Step 4 — Core services (all behind interfaces) +IStreamMonitorService / StreamMonitorService + +Wraps TwitchApiClient +CheckIsLiveAsync(string username) → LiveStreamInfo? +GetLatestVodAsync(string username) → VodInfo? +IRecorderService / RecorderService + +StartRecordingAsync(string username, string quality, string outputPath, CancellationToken ct) → Task +Invokes streamlink via ProcessRunner +Passes --hls-live-restart, --stream-segment-threads, optional OAuth header +Returns when streamlink exits (either stream ended or ct cancelled) +IProcessorService / ProcessorService + +ProcessRawStreamAsync(string rawPath, string outputPath, EffectiveConfig cfg, CancellationToken ct) +Builds ffmpeg args: hwaccel, thread count, error recovery flags, faststart, copy codecs +MergeVideoChatAsync(string videoPath, string chatVideoPath, string outputPath, string layout, CancellationToken ct) +IDownloaderService / DownloaderService + +DownloadVodAsync(VodInfo vod, string outputPath, EffectiveConfig cfg, CancellationToken ct) → bool +Invokes TwitchDownloaderCLI videodownload +Chat download methods stubbed with NotImplementedException / commented structure; interface is defined now to keep the architecture clean +IUploadService / UploadService + +UploadAsync(string localRoot, IEnumerable relativeFilePaths, string rcloneDest, CancellationToken ct) → bool +Writes a temp files-from list, invokes rclone copy --files-from +Returns success/failure; preserves local files on failure +IFileManagerService / FileManagerService + +InitializeDirectories(string rootPath, string username) +GetPaths(string rootPath, string username, string filenameBase) → ArchivePaths record (all expected paths) +CleanRawFile(string path, bool cleanRaw) +DeleteLocalFiles(ArchivePaths paths, EffectiveConfig cfg) +GetUniquePath(string path) → adds numeric suffix if file exists +Step 5 — Recording resilience engine +RecoveryPolicy (POCO, unit-testable, no DI deps): + +Encodes a state machine with these states: + +State Meaning +Monitoring Normal polling at refresh interval +Recording streamlink subprocess active +FastReconnect Stream ended; checking every 10 s for up to 2 minutes +SlowReconnect Still not live after 2 min; checking every 60 s concurrently with post-processing +PostProcessing Confirmed ended; ffmpeg / VOD download / upload running +NetworkFault Twitch API unreachable; exponential back-off (30 s → capped at 10 min) +Transitions: + +Recording → streamlink exits → enter FastReconnect, record phase start time +FastReconnect → live confirmed → start new Recording (new filename/segment) +FastReconnect (2 min elapsed) → enter SlowReconnect + kick off PostProcessing concurrently +SlowReconnect → live confirmed → start new Recording +SlowReconnect / Monitoring → API call throws network error → enter NetworkFault +NetworkFault → successful API response → return to previous state (Monitoring or re-enter FastReconnect if we were mid-reconnect) +NetworkFault backoff: 30s * 2^attempt, capped at 600s +RecoveryPolicy is a pure class with a Tick(DateTime now, bool? isLive, bool networkError) method → returns RecoveryDecision (what to do next + sleep duration). Fully unit-testable with no async or DI. + +StreamWorker : BackgroundService + +One instance per enabled streamer +Holds RecoveryPolicy instance +Main loop: evaluate policy decision → execute the corresponding service call → loop +Started/stopped by StreamWorkerManager +Writes job records to SQLite on start/complete/fail +StreamWorkerManager + +StartWorker(string username), StopWorker(string username), RestartWorker(string username) +Called at app startup for all enabled streamers +Called from Web UI on enable/disable/config change +Workers stored in ConcurrentDictionary +Step 6 — Persistence (SQLite + EF Core) +ArchiveDbContext with three tables: + +StreamSessions: Id, StreamerUsername, TwitchStreamId, Title, StartedAt, EndedAt, Status (Recording/Processing/Uploading/Complete/Failed) + +ArchiveJobs: Id, SessionId, JobType (enum: RecordLive, ProcessLive, DownloadVod, ProcessVod, UploadCloud, DeleteLocal), Status, StartedAt, CompletedAt, FilePath, ErrorMessage + +StreamerStates: Username, IsMonitoring, LastCheckedAt, CurrentRecoveryState + +Migrations via EF Core CLI. ISessionRepository / IJobRepository interfaces for testability with in-memory EF provider in tests. + +Step 7 — Process output streaming +IProcessOutputStore: + +AppendLine(string streamerId, Guid jobId, string line, bool isError) +GetRecentLines(string streamerId, Guid jobId, int count = 500) → IReadOnlyList +In-memory circular buffer (1000 lines per job, last 20 jobs per streamer) +ProcessOutputHub : Hub (SignalR): + +Clients call SubscribeToStreamer(string username) → join group streamer:{username} +Clients call SubscribeToJob(Guid jobId) → join group job:{jobId} +Server pushes ReceiveLine(OutputLine line) from ProcessRunner via IHubContext +On subscribe: server immediately sends buffered lines from IProcessOutputStore +Step 8 — Blazor Server Web UI +Authentication: Cookie-based single-password auth via ASP.NET Core minimal auth middleware. Password stored as BCrypt hash in AppSettings. Login.razor page at /login. Protected routes with [Authorize]. + +Pages & Components: + +Dashboard.razor (/) — grid of all configured streamers showing: username, live/offline badge, current recovery state, last recorded session, quick Start/Stop monitoring toggle + +StreamerDetail.razor (/streamer/{username}) — live status, current job pipeline steps (record → process → upload with progress), ProcessOutputConsole.razor showing real-time terminal output via SignalR + +ProcessOutputConsole.razor — reusable Blazor component; subscribes to SignalR on mount, renders an auto-scrolling
 with colored output (stdout = white, stderr = orange/red), handles reconnect
+
+Sessions.razor (/sessions) — paginated list of past archive sessions with job statuses and expandable per-job output
+
+GlobalConfig.razor (/config/global) — EditForm bound to GlobalConfig model with data annotations validation, Save button calls IConfigurationService.SaveGlobal()
+
+StreamerConfig.razor (/config/{username}) — similar form for per-streamer overrides; each field has a nullable toggle (inherit from global vs override)
+
+AddStreamer.razor (/config/new) — minimal form: username + enabled; creates new config/streamers/{username}.json
+
+AppSettings.razor (/settings) — tool paths (streamlink, ffmpeg, TwitchDownloaderCLI, rclone), change password
+
+Step 9 — NLog configuration
+nlog.config (XML): two targets:
+
+Console (colored, with level formatting)
+File rolling (logs/archive-${shortdate}.log, keep 30 days)
+Log structured context: StreamerUsername, JobId, JobType as NLog ScopeContext properties. Service methods open a scope via ILogger.BeginScope(...).
+
+Step 10 — Docker
+Dockerfile (multi-stage):
+
+Build stage: mcr.microsoft.com/dotnet/sdk:10.0
+Runtime stage: mcr.microsoft.com/dotnet/aspnet:10.0 (Linux)
+Install ffmpeg, streamlink (via pip), download TwitchDownloaderCLI binary for linux-x64
+rclone installed via shell script or apt
+Expose port 8080
+ENTRYPOINT ["dotnet", "TwitchArchive.Web.dll"]
+docker-compose.yml:
+
+Volume mounts: ./config:/app/config, ./archive:/app/archive, ./logs:/app/logs
+Environment variables: CLIENT-ID, CLIENT-SECRET, OAUTH-PRIVATE-TOKEN
+Windows dev: run directly with dotnet run; tool paths auto-detected (Windows vs Linux) via RuntimeInformation.IsOSPlatform(OSPlatform.Windows) in ToolPathResolver
+
+Step 11 — Unit tests
+TwitchArchive.Tests covers:
+
+RecoveryPolicyTests — state machine transitions, timing phases, network fault backoff; pure synchronous tests
+ConfigurationServiceTests — JSON load/save/merge with temp files
+TwitchApiClientTests — mocked HttpMessageHandler; OAuth, GQL queries, 401 refresh, network errors
+FileManagerServiceTests — path generation, directory creation with temp directories
+DownloaderServiceTests / RecorderServiceTests — mocked ProcessRunner; verify correct CLI arguments
+UploadServiceTests — mocked ProcessRunner; verify rclone argument construction
+SessionRepositoryTests — EF Core in-memory provider
+EffectiveConfigTests — global + streamer override merge logic
+Verification
+dotnet build dotnet/TwitchArchive.sln — zero warnings, zero errors
+dotnet test dotnet/TwitchArchive.Tests/ — all tests green
+docker compose up --build in the dotnet/ folder → app reachable at http://localhost:8080
+Manual: add a test streamer config, enable monitoring, confirm it polls and records a live stream, runs ffmpeg, uploads via rclone
+Resilience manual test: kill network during recording → verify FastReconnect phase kicks in, resumes after connectivity restored
+Decisions
+
+Blazor Server chosen for real-time terminal output without a separate API; no WASM needed
+streamlink for live + TwitchDownloaderCLI for VOD (same split as Python; streamlink gives better live resilience)
+Simple BCrypt password auth (not full Identity — this is a single-user tool)
+RecoveryPolicy as a pure POCO state machine keeps the resilience logic fully unit-testable without async/mocking
+Polly WaitAndRetryForever on HttpClient handles persistent network failure independently of the application-level state machine; they are complementary — Polly handles individual HTTP call retries, RecoveryPolicy handles the overall workflow state
+Chat download service interface is defined in Step 4 but methods are stubbed — adding implementation later requires only filling in DownloaderService without touching any other layer
+Config files remain in the same format/location so the Python and C# versions can share the same config directory
\ No newline at end of file
diff --git a/UpgradePlan2.md b/UpgradePlan2.md
new file mode 100644
index 0000000..cc4ffbd
--- /dev/null
+++ b/UpgradePlan2.md
@@ -0,0 +1,78 @@
+
+
+
@Body
``` Add a top bar (`
`) with the app title "Twitch Archive" and a hamburger toggle ` + + +@code { + private TwitchArchive.Core.Config.StreamerConfig model = new() { Enabled = true }; + + private void Save() + { + model.Username = model.Username?.Trim().ToLowerInvariant() ?? string.Empty; + if (string.IsNullOrWhiteSpace(model.Username)) return; + ConfigService.SaveStreamer(model); + Nav.NavigateTo($"/config/{model.Username}"); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor new file mode 100644 index 0000000..9ccb7d3 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/AppSettings.razor @@ -0,0 +1,92 @@ +@page "/settings" +@using System.Text.Json +@inject TwitchArchive.Web.Services.IAuthService Auth + +

App Settings

+ +@if (saved) +{ +
Saved.
+} + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ +

Change Password

+@if (!string.IsNullOrEmpty(pwError)) +{ +
@pwError
+} +
+ + + + +
+ +@code { + private TwitchArchive.Core.Config.AppSettings model = new(); + private bool saved = false; + private string currentPw = string.Empty; + private string newPw = string.Empty; + private string confirmPw = string.Empty; + private string pwError = string.Empty; + + protected override void OnInitialized() + { + Load(); + } + + private void Load() + { + var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + try + { + if (!File.Exists(file)) { model = new(); return; } + var txt = File.ReadAllText(file); + model = JsonSerializer.Deserialize(txt) ?? new TwitchArchive.Core.Config.AppSettings(); + } + catch { model = new(); } + } + + private void Save() + { + var file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + try + { + var txt = JsonSerializer.Serialize(model, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(file, txt); + saved = true; + Auth.Refresh(); + } + catch { } + } + + private void ChangePassword() + { + pwError = string.Empty; + if (!Auth.ValidatePassword(currentPw)) { pwError = "Current password incorrect"; return; } + if (string.IsNullOrWhiteSpace(newPw)) { pwError = "New password required"; return; } + if (newPw != confirmPw) { pwError = "Passwords do not match"; return; } + var hash = BCrypt.Net.BCrypt.HashPassword(newPw); + Auth.SetPasswordHash(hash); + pwError = string.Empty; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor new file mode 100644 index 0000000..96ba67d --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/GlobalConfig.razor @@ -0,0 +1,45 @@ +@page "/config/global" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService + +

Global Configuration

+ +@if (saved) +{ +
Saved.
+} + + +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ +@code { + private TwitchArchive.Core.Config.GlobalConfig model = new(); + private bool saved = false; + + protected override void OnInitialized() + { + model = ConfigService.LoadGlobal(); + } + + private void Save() + { + ConfigService.SaveGlobal(model); + saved = true; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Index.razor b/dotnet/src/TwitchArchive.Web/Pages/Index.razor new file mode 100644 index 0000000..9120fe2 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Index.razor @@ -0,0 +1,64 @@ +@page "/" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Web.Services.SessionCacheService SessionCache + +

Dashboard

+ +
+ @foreach (var s in streamers) + { +
+
+ @s + @(WorkerManager.IsRunning(s) ? "Live" : "Offline") +
+
+
Last session: @(lastStarts.ContainsKey(s) ? lastStarts[s].ToLocalTime().ToString() : "-")
+
+ + +
+
+
+ } +
+ +@code { + private List streamers = new(); + private Dictionary lastStarts = new(); + + private void OnCacheUpdatedHandler() + { + _ = InvokeAsync(() => { + lastStarts = SessionCache.GetSnapshot(); + StateHasChanged(); + }); + } + + protected override async Task OnInitializedAsync() + { + LoadStreamers(); + lastStarts = SessionCache.GetSnapshot(); + SessionCache.Updated += OnCacheUpdatedHandler; + } + + private void LoadStreamers() + { + var cfgDir = Path.Combine(Environment.CurrentDirectory, "config", "streamers"); + if (Directory.Exists(cfgDir)) + { + streamers = Directory.GetFiles(cfgDir, "*.json").Select(f => Path.GetFileNameWithoutExtension(f)).ToList(); + } + } + + // Index reads from the singleton SessionCacheService; updates are pushed via the Updated event. + + private void Start(string u) { WorkerManager.StartWorker(u); } + private async Task Stop(string u) { await WorkerManager.StopWorkerAsync(u); } + + public async ValueTask DisposeAsync() + { + SessionCache.Updated -= OnCacheUpdatedHandler; + await Task.CompletedTask; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Login.razor b/dotnet/src/TwitchArchive.Web/Pages/Login.razor new file mode 100644 index 0000000..ff4f1da --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Login.razor @@ -0,0 +1,20 @@ +@page "/login" +@attribute [AllowAnonymous] +@using Microsoft.AspNetCore.Components + +

Login

+ +@if (error) +{ +
Invalid password
+} + +
+ + +
+ +@code { + [Parameter] + public bool error { get; set; } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor new file mode 100644 index 0000000..4cd79de --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Monitor.razor @@ -0,0 +1,37 @@ +@page "/monitor" +@inject TwitchArchive.Core.Workers.StreamWorkerManager WorkerManager +@inject TwitchArchive.Core.Services.IProcessOutputStore OutputStore +@using TwitchArchive.Web.Shared + +

Streamer Monitor

+ +
+ + + +
+ +
+ Status: @status +
+ + + +@code { + private string username = "hackerling"; + private string status = "idle"; + + private void Start() + { + if (string.IsNullOrWhiteSpace(username)) return; + WorkerManager.StartWorker(username); + status = WorkerManager.IsRunning(username) ? "running" : "starting"; + } + + private async Task Stop() + { + if (string.IsNullOrWhiteSpace(username)) return; + await WorkerManager.StopWorkerAsync(username); + status = WorkerManager.IsRunning(username) ? "running" : "stopped"; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor new file mode 100644 index 0000000..256da43 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/Sessions.razor @@ -0,0 +1,73 @@ +@page "/sessions" +@using TwitchArchive.Core.Persistence.Models +@inject TwitchArchive.Core.Persistence.ISessionRepository SessionRepo + +

Sessions

+ + + + + + + + + @foreach (var s in sessions) + { + + + + + + + + + @if (expandedSession == s.Id) + { + + } + } + +
IdStreamerStartedEndedStatusJobs
@s.Id@s.StreamerUsername@s.StartedAt.ToLocalTime()@(s.EndedAt?.ToLocalTime().ToString() ?? "-")@s.Status
+
    + @if (jobs?.Any() ?? false) + { + @foreach (var j in jobs) + { +
  • @j.Id - @j.JobType - @j.Status - @j.StartedAt.ToLocalTime() - @(j.FilePath ?? "")
  • + } + } + else + { +
  • No jobs
  • + } +
+
+ +@code { + private List sessions = new(); + private List? jobs; + private long? expandedSession; + + protected override async Task OnInitializedAsync() + { + await Refresh(); + } + + private async Task Refresh() + { + sessions = await SessionRepo.GetRecentSessionsAsync(50); + StateHasChanged(); + } + + private async Task ToggleJobs(long sessionId) + { + if (expandedSession == sessionId) + { + expandedSession = null; + jobs = null; + return; + } + jobs = await SessionRepo.GetJobsForSessionAsync(sessionId); + expandedSession = sessionId; + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor new file mode 100644 index 0000000..bf0c72a --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerConfig.razor @@ -0,0 +1,71 @@ +@page "/config/{Username}" +@inject TwitchArchive.Core.Config.IConfigurationService ConfigService +@inject NavigationManager Nav + +

Streamer Config: @Username

+ + +
+ + +
+ +
+ + Override + +
+ +
+ + Override + +
+ +
+ + +
+ +
+ + Override + +
+ + + +
+ +@code { + [Parameter] + public string Username { get; set; } = string.Empty; + + private TwitchArchive.Core.Config.StreamerConfig model = new(); + private TwitchArchive.Core.Config.GlobalConfig? global; + private bool overrideQuality = false; + private bool overrideUpload = false; + private bool overrideStreamlink = false; + + protected override void OnInitialized() + { + global = ConfigService.LoadGlobal(); + var s = ConfigService.LoadStreamer(Username); + if (s != null) model = s; + } + + private void Save() + { + model.Username = Username; + if (!overrideQuality) model.Quality = null; + if (!overrideUpload) model.UploadToCloud = null; + if (!overrideStreamlink) model.StreamlinkPath = null; + ConfigService.SaveStreamer(model); + } + + private void Delete() + { + ConfigService.DeleteStreamer(Username); + Nav.NavigateTo("/"); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor new file mode 100644 index 0000000..bafa148 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/StreamerDetail.razor @@ -0,0 +1,22 @@ +@page "/streamer/{Username}" +@using TwitchArchive.Core.Workers +@inject StreamWorkerManager WorkerManager + +

Streamer: @Username

+ +
+ Status: @(WorkerManager.IsRunning(Username) ? "Live" : "Offline") +
+ +
+
Record
+
Process
+
Upload
+
+ + + +@code { + [Parameter] + public string Username { get; set; } = string.Empty; +} diff --git a/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml new file mode 100644 index 0000000..1463d9e --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Pages/_Host.cshtml @@ -0,0 +1,19 @@ +@page "/" +@namespace TwitchArchive.Web.Pages +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers + + + + + + Twitch Archive + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/Program.cs b/dotnet/src/TwitchArchive.Web/Program.cs new file mode 100644 index 0000000..d655b3e --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Program.cs @@ -0,0 +1,104 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.EntityFrameworkCore; +using TwitchArchive.Core.Services; +using TwitchArchive.Core.Config; +using TwitchArchive.Core.Persistence; +using TwitchArchive.Web.Hubs; +using TwitchArchive.Web.Services; + +var builder = WebApplication.CreateBuilder(args); + +// Add services +builder.Services.AddRazorPages(); +builder.Services.AddServerSideBlazor(); +builder.Services.AddSignalR(); + +// Authentication +builder.Services.AddSingleton(); +builder.Services.AddAuthentication(options => +{ + options.DefaultScheme = Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme; +}) +.AddCookie(options => { options.LoginPath = "/login"; }); + +// Register Core services +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHttpClient(client => { /* base config if needed */ }); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// Configuration service for global + per-streamer JSON files in ./config +builder.Services.AddScoped(); + +// Broadcaster forwards output store events to the SignalR hub +builder.Services.AddSingleton(); + +// SQLite DB (file in app folder) +var conn = "Data Source=archive.db"; +// Provide a factory for creating DbContext instances for background work +builder.Services.AddDbContextFactory(opt => opt.UseSqlite(conn)); +// persistence +builder.Services.AddScoped(); + +// Session cache and background refresh +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + +var app = builder.Build(); + +if (!app.Environment.IsDevelopment()) +{ + app.UseExceptionHandler("/Error"); +} + +// Ensure DB schema exists (creates DB when missing) +using (var scope = app.Services.CreateScope()) +{ + var factory = scope.ServiceProvider.GetRequiredService>(); + try + { + using var db = factory.CreateDbContext(); + db.Database.EnsureCreated(); + } + catch { } +} + +app.UseStaticFiles(); +app.UseRouting(); + +app.UseAuthentication(); +app.UseAuthorization(); + +// login endpoint +app.MapPost("/auth/login", async (Microsoft.AspNetCore.Http.HttpContext http, TwitchArchive.Web.Services.IAuthService auth) => +{ + try + { + var form = await http.Request.ReadFormAsync(); + var pwd = form["password"].ToString(); + if (auth.ValidatePassword(pwd)) + { + var claims = new[] { new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "admin") }; + var id = new System.Security.Claims.ClaimsIdentity(claims, Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme); + var principal = new System.Security.Claims.ClaimsPrincipal(id); + await http.SignInAsync(Microsoft.AspNetCore.Authentication.Cookies.CookieAuthenticationDefaults.AuthenticationScheme, principal); + http.Response.Redirect("/"); + return; + } + } + catch { } + http.Response.Redirect("/login?error=1"); +}); + +app.MapBlazorHub(); +app.MapHub("/processOutputHub"); +app.MapFallbackToPage("/_Host"); + +app.Run(); diff --git a/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json new file mode 100644 index 0000000..9c88f37 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "TwitchArchive.Web": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:64466;http://localhost:64467" + } + } +} \ No newline at end of file diff --git a/dotnet/src/TwitchArchive.Web/Services/AuthService.cs b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs new file mode 100644 index 0000000..12f4e2f --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/AuthService.cs @@ -0,0 +1,50 @@ +using System; +using System.IO; +using System.Text.Json; +using TwitchArchive.Core.Config; + +namespace TwitchArchive.Web.Services +{ + public class AuthService : IAuthService + { + private readonly string _file; + private AppSettings _settings = new(); + + public AuthService() + { + _file = Path.Combine(AppContext.BaseDirectory, "appsettings.json"); + Refresh(); + } + + public void Refresh() + { + try + { + if (!File.Exists(_file)) { _settings = new AppSettings(); return; } + var txt = File.ReadAllText(_file); + _settings = JsonSerializer.Deserialize(txt) ?? new AppSettings(); + } + catch { _settings = new AppSettings(); } + } + + public bool ValidatePassword(string plain) + { + if (string.IsNullOrWhiteSpace(_settings.PasswordHash)) return true; + try { return BCrypt.Net.BCrypt.Verify(plain ?? string.Empty, _settings.PasswordHash); } + catch { return false; } + } + + public void SetPasswordHash(string hash) + { + try + { + if (string.IsNullOrWhiteSpace(_file)) return; + _settings.PasswordHash = hash; + var txt = JsonSerializer.Serialize(_settings, new JsonSerializerOptions { WriteIndented = true }); + File.WriteAllText(_file, txt); + Refresh(); + } + catch { } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs new file mode 100644 index 0000000..cd3b0fd --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/IAuthService.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; + +namespace TwitchArchive.Web.Services +{ + public interface IAuthService + { + bool ValidatePassword(string plain); + void Refresh(); + void SetPasswordHash(string hash); + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs new file mode 100644 index 0000000..641ddf9 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/ProcessOutputBroadcaster.cs @@ -0,0 +1,35 @@ +using Microsoft.AspNetCore.SignalR; +using System; +using TwitchArchive.Core.Services; +using TwitchArchive.Web.Hubs; + +namespace TwitchArchive.Web.Services +{ + public class ProcessOutputBroadcaster : IDisposable + { + private readonly IProcessOutputStore _store; + private readonly IHubContext _hub; + + public ProcessOutputBroadcaster(IProcessOutputStore store, IHubContext hub) + { + _store = store; + _hub = hub; + _store.LineAppended += OnLineAppended; + } + + private void OnLineAppended(string streamer, OutputLine line) + { + try + { + // send to clients; clients should listen on "ReceiveLine" + _hub.Clients.Group(streamer).SendAsync("ReceiveLine", streamer, line); + } + catch { } + } + + public void Dispose() + { + _store.LineAppended -= OnLineAppended; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs new file mode 100644 index 0000000..abe7855 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionCacheService.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using TwitchArchive.Core.Persistence.Models; + +namespace TwitchArchive.Web.Services +{ + public class SessionCacheService + { + private readonly object _lock = new(); + private Dictionary _snapshot = new(); + + public event Action? Updated; + + public void Update(IEnumerable sessions) + { + var next = new Dictionary(); + foreach (var s in sessions) + { + if (!next.ContainsKey(s.StreamerUsername)) next[s.StreamerUsername] = s.StartedAt; + } + + lock (_lock) + { + _snapshot = next; + } + + Updated?.Invoke(); + } + + public Dictionary GetSnapshot() + { + lock (_lock) + { + return new Dictionary(_snapshot); + } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs new file mode 100644 index 0000000..93523b8 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Services/SessionRefreshHostedService.cs @@ -0,0 +1,50 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using TwitchArchive.Core.Persistence; + +namespace TwitchArchive.Web.Services +{ + public class SessionRefreshHostedService : BackgroundService + { + private readonly IServiceScopeFactory _scopeFactory; + private readonly SessionCacheService _cache; + private readonly ILogger _logger; + private readonly TimeSpan _interval; + + public SessionRefreshHostedService(IServiceScopeFactory scopeFactory, SessionCacheService cache, ILogger logger) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _interval = TimeSpan.FromSeconds(10); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_interval); + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + using var scope = _scopeFactory.CreateScope(); + var repo = scope.ServiceProvider.GetRequiredService(); + var sessions = await repo.GetRecentSessionsAsync(200, stoppingToken).ConfigureAwait(false); + _cache.Update(sessions); + } + catch (OperationCanceledException) { break; } + catch (Exception ex) + { + _logger.LogError(ex, "Error while refreshing sessions"); + } + } + } + catch (OperationCanceledException) { } + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor new file mode 100644 index 0000000..dd8a426 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/MainLayout.razor @@ -0,0 +1,21 @@ +@inherits LayoutComponentBase +
+ +

Twitch Archive

+
+
+ +
+ @Body +
+
+ +@code { + bool sidebarCollapsed; + void ToggleSidebar() => sidebarCollapsed = !sidebarCollapsed; +} diff --git a/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor new file mode 100644 index 0000000..f5c2c76 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/Shared/ProcessConsole.razor @@ -0,0 +1,93 @@ +@using TwitchArchive.Core.Services +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.SignalR.Client +@using System.Linq +@inject IProcessOutputStore OutputStore +@inject NavigationManager Navigation + +@inherits ComponentBase +
+ @foreach (var line in lines) + { +
@line.TimestampUtc.ToLocalTime().ToString("HH:mm:ss") - @line.Line
+ } +
+ +@code { + [Parameter] + public string Streamer { get; set; } = string.Empty; + + private List lines = new(); + private HubConnection? hubConnection; + private string? _currentGroup; + + protected override async Task OnInitializedAsync() + { + OutputStore.LineAppended += OnLineAppended; + + hubConnection = new HubConnectionBuilder() + .WithUrl(Navigation.ToAbsoluteUri("/processOutputHub")) + .WithAutomaticReconnect() + .Build(); + + hubConnection.On("ReceiveLine", (streamer, line) => + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + // avoid duplicates + if (lines.Any(l => l.TimestampUtc == line.TimestampUtc && l.Line == line.Line)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + }); + + await hubConnection.StartAsync(); + if (!string.IsNullOrWhiteSpace(Streamer)) + { + await hubConnection.SendAsync("JoinStreamerGroup", Streamer); + _currentGroup = Streamer; + } + + await base.OnInitializedAsync(); + } + + protected override async Task OnParametersSetAsync() + { + if (hubConnection == null) return; + if (!string.IsNullOrWhiteSpace(_currentGroup) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); } catch { } + _currentGroup = null; + } + + if (!string.IsNullOrWhiteSpace(Streamer) && !string.Equals(_currentGroup, Streamer, StringComparison.OrdinalIgnoreCase)) + { + try { await hubConnection.SendAsync("JoinStreamerGroup", Streamer); _currentGroup = Streamer; } catch { } + } + + await base.OnParametersSetAsync(); + } + + private void OnLineAppended(string streamer, OutputLine line) + { + if (!string.Equals(streamer, Streamer, StringComparison.OrdinalIgnoreCase)) return; + lines.Add(line); + if (lines.Count > 1000) lines.RemoveAt(0); + InvokeAsync(StateHasChanged); + } + + public async ValueTask DisposeAsync() + { + OutputStore.LineAppended -= OnLineAppended; + if (hubConnection != null) + { + try + { + if (!string.IsNullOrWhiteSpace(_currentGroup)) await hubConnection.SendAsync("LeaveStreamerGroup", _currentGroup); + } + catch { } + try { await hubConnection.StopAsync(); } catch { } + try { await hubConnection.DisposeAsync(); } catch { } + hubConnection = null; + } + } +} diff --git a/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj new file mode 100644 index 0000000..774299b --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/TwitchArchive.Web.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/dotnet/src/TwitchArchive.Web/_Imports.razor b/dotnet/src/TwitchArchive.Web/_Imports.razor new file mode 100644 index 0000000..34b6706 --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/_Imports.razor @@ -0,0 +1,10 @@ +@using System +@using System.Net.Http +@using Microsoft.AspNetCore.Components +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Authorization +@using TwitchArchive.Core +@using TwitchArchive.Core.Services +@using TwitchArchive.Web.Shared diff --git a/dotnet/src/TwitchArchive.Web/archive.db b/dotnet/src/TwitchArchive.Web/archive.db new file mode 100644 index 0000000..4e86411 Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-shm b/dotnet/src/TwitchArchive.Web/archive.db-shm new file mode 100644 index 0000000..08698a3 Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db-shm differ diff --git a/dotnet/src/TwitchArchive.Web/archive.db-wal b/dotnet/src/TwitchArchive.Web/archive.db-wal new file mode 100644 index 0000000..f7800fa Binary files /dev/null and b/dotnet/src/TwitchArchive.Web/archive.db-wal differ diff --git a/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css new file mode 100644 index 0000000..ad8771f --- /dev/null +++ b/dotnet/src/TwitchArchive.Web/wwwroot/css/app.css @@ -0,0 +1,16 @@ +/* App layout styles for Twitch Archive */ +.page { display:flex; height:100vh; } +.sidebar { width:220px; flex-shrink:0; background:#1e1e2e; color:#cdd6f4; overflow-y:auto; } +.sidebar.collapsed { display:none; } +.main { flex:1; overflow-y:auto; padding:1.5rem; } +.nav-link { display:block; padding:0.6rem 1rem; color:#cdd6f4; text-decoration:none; } +.nav-link.active { background:#313244; border-left:3px solid #89b4fa; } +.topbar { display:none; background:#111; color:#fff; padding:0.6rem 1rem; align-items:center; } +.topbar .hamburger { font-size:1.2rem; margin-right:1rem; background:transparent; border:none; color:inherit; } +@media(max-width:768px) { + .sidebar { display:none; } + .topbar { display:flex; } +} +@media(min-width:769px) { + .topbar { display:none; } +} diff --git a/only-vod-chat.py b/only-vod-chat.py index b3a948d..4d9a92d 100644 --- a/only-vod-chat.py +++ b/only-vod-chat.py @@ -228,7 +228,11 @@ class TwitchArchive: elif self.os == 'linux': subprocess.call([bin_path+"/TwitchDownloaderCLI", "chatupdate", "-i", chat_json_path, "-o", chat_html_path, "-E", "--temp-path", f"{bin_path}/temp"]) if self.username == 'KalathrasLolweapon': print('Uploading html chat to b2 bucket') - subprocess.call(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', chat_html_path, 'b2:kala-help/chat_html', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() except Exception as e: print('A ERROR has ocurred and chat will need to be updated to html manually') @@ -236,9 +240,22 @@ class TwitchArchive: print('Uploading files:') if self.os == 'windows': if self.username == 'KalathrasLolweapon': - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) - subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress']) - else:subprocess.call(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress']) + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/Chat', 'GD:Chat', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() + else: + proc = subprocess.Popen(['rclone', 'copy', str(pathlib.Path(__file__).parent.resolve())+'/VODS', 'GD:VODS', '--progress'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + if proc.stdout: + for line in proc.stdout: + print(line, end='') + proc.wait() elif self.os == 'linux':subprocess.call([bin_path+'/upload.sh', str(pathlib.Path(self.root_path).resolve()),self.username]) if self.deleteFiles == 1: diff --git a/twitch-archive.py b/twitch-archive.py deleted file mode 100644 index 2fdcb4e..0000000 --- a/twitch-archive.py +++ /dev/null @@ -1,1565 +0,0 @@ -""" -Twitch Archive - Automated Twitch Stream Recording & Archiving System - -This script monitors a Twitch channel and automatically: -- Records live streams as they happen -- Downloads VODs (Video on Demand) after the stream ends -- Downloads and renders chat logs -- Saves stream metadata -- Uploads everything to cloud storage (optional) - -Requirements: -- Python 3.7+ -- External tools: streamlink, ffmpeg, TwitchDownloaderCLI, rclone (optional) -- Configuration file: config.json (copy from config.sample.json) -- Environment file: .env (for API credentials) - -Refactored Version 2.0: -This version has been split into multiple modules for better maintainability: -- modules/constants.py: Constants and default configuration -- modules/config.py: Configuration management -- modules/notifications.py: Email notifications -- modules/utils.py: Utility functions -- modules/stream_monitor.py: Stream monitoring and API -- modules/recorder.py: Live stream recording -- modules/processor.py: Video/audio processing -- modules/downloader.py: VOD and chat downloading -- modules/file_manager.py: File and cloud management -""" - -# Standard library imports -import os -import sys -import time -import json -import signal -import getopt -from typing import Dict, Optional, Any -from datetime import datetime, timedelta - -# Third-party imports -from colorama import Fore, Style -from pytz import timezone -from dotenv import load_dotenv, find_dotenv - -# Local module imports -from modules.constants import DEFAULT_CONFIG, PREFIX_LIVE, PREFIX_VOD, PREFIX_CHAT, PREFIX_MERGED, PREFIX_METADATA -from modules.config import ConfigManager -from modules.notifications import NotificationManager -from modules.utils import ( - detect_operating_system, get_ffmpeg_executable, get_twitch_downloader_executable, - get_unique_filename, get_video_duration, verify_streamlink, verify_ffmpeg, verify_twitch_downloader -) -from modules.stream_monitor import StreamMonitor -from modules.recorder import StreamRecorder -from modules.processor import StreamProcessor -from modules.downloader import ContentDownloader -from modules.file_manager import FileManager - - -class TwitchArchive: - """ - Main class for the Twitch Archive system. - - Handles monitoring a Twitch channel, recording live streams, and downloading - VODs, chat logs, and metadata. Can optionally upload to cloud storage. - - Refactored Version 2.0: This class now delegates most functionality to - specialized modules for better code organization. - """ - - def __init__(self, config: Optional[Dict[str, Any]] = None): - """ - Initialize the TwitchArchive with configuration settings. - - Args: - config: Configuration dictionary. If None, loads from legacy config.json - """ - if config is None: - # Legacy mode: load from config.json - self.load_config() - else: - # New mode: use provided config - for key, value in config.items(): - setattr(self, key, value) - - # Initialize system components - self.os_type = detect_operating_system() - self.shutdown_requested = False - self.current_stream_data = {} - - # Initialize component modules (created during run()) - self.stream_monitor = None - self.notification_manager = None - self.file_manager = None - self.recorder = None - self.processor = None - self.downloader = None - - def load_config(self) -> None: - """ - Load configuration from config.json file (legacy support). - - Falls back to default configuration if file is not found or cannot be read. - Filters out comment fields (starting with '_') from the config. - """ - config_file = os.path.join(os.path.dirname(__file__), 'config.json') - - # Start with default configuration - config = DEFAULT_CONFIG.copy() - - # Try to load and merge user configuration - if os.path.exists(config_file): - try: - with open(config_file, 'r', encoding='utf-8') as f: - user_config = json.load(f) - # Filter out comment fields (those starting with '_') - user_config = {k: v for k, v in user_config.items() if not k.startswith('_')} - # Merge user config with defaults (user config takes precedence) - config.update(user_config) - print(f'{Fore.GREEN}✓ Configuration loaded from config.json{Style.RESET_ALL}') - except json.JSONDecodeError as e: - print(f'{Fore.YELLOW}⚠ Warning: Invalid JSON in config.json: {e}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.YELLOW}⚠ Warning: Could not load config.json: {e}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Using default configuration{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Warning: config.json not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Copy config.sample.json to config.json and edit it with your settings{Style.RESET_ALL}') - - # Set all configuration values as instance attributes - for key, value in config.items(): - setattr(self, key, value) - - def _load_environment_variables(self) -> None: - """ - Load environment variables from .env file. - - Required variables: - - CLIENT-ID: Twitch API client ID - - CLIENT-SECRET: Twitch API client secret - - OAUTH-PRIVATE-TOKEN: Optional, for accessing subscriber-only streams - - SENDER: Email address for notifications (if enabled) - - RECEIVER: Email address to receive notifications (if enabled) - - PASSWD: Email password for sending notifications (if enabled) - - Raises: - SystemExit: If .env file is not found - """ - if not load_dotenv(find_dotenv()): - print(f'{Fore.RED}✗ ERROR: .env file not found{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Create a .env file with your Twitch API credentials{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Required: CLIENT-ID, CLIENT-SECRET{Style.RESET_ALL}') - sys.exit(1) - - def _initialize_components(self) -> None: - """Initialize all component modules.""" - # Stream monitoring - self.stream_monitor = StreamMonitor(self.username) - - # Notifications - self.notification_manager = NotificationManager( - enabled=self.notifications, - username=self.username - ) - - # File management - self.file_manager = FileManager( - root_path=self.root_path, - username=self.username, - config=vars(self) - ) - self.file_manager.initialize_directories() - - # Recording - self.recorder = StreamRecorder( - username=self.username, - quality=self.quality, - refresh=self.refresh, - hls_segments=self.hls_segments, - streamlink_ttvlol=self.streamlink_ttvlol, - shutdown_callback=lambda: self.shutdown_requested - ) - - # Processing - ffmpeg_path = get_ffmpeg_executable(self.os_type) - self.processor = StreamProcessor( - os_type=self.os_type, - ffmpeg_path=ffmpeg_path, - config=vars(self) - ) - - # Downloading - twitch_downloader_path = get_twitch_downloader_executable(self.os_type) - self.downloader = ContentDownloader( - twitch_downloader_path=twitch_downloader_path, - ffmpeg_path=ffmpeg_path, - config=vars(self) - ) - - def _print_configuration_summary(self) -> None: - """Print a summary of the current configuration to the console.""" - print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.CYAN}TWITCH ARCHIVE - Configuration Summary{Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') - - # Basic settings - print(f'Streamer: {Fore.GREEN}{self.username}{Style.RESET_ALL}') - print(f'Quality: {Fore.GREEN}{self.quality}{Style.RESET_ALL}') - print(f'Storage: {Fore.GREEN}{os.path.abspath(self.root_path)}{Style.RESET_ALL}') - print(f'Refresh rate: {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}\n') - - # Feature toggles - self._print_toggle('Email notifications', self.notifications) - self._print_toggle('Metadata download', self.downloadMETADATA) - self._print_toggle('VOD download', self.downloadVOD) - self._print_toggle('Chat download & render', self.downloadCHAT) - if self.downloadCHAT: - self._print_toggle(' ↳ Merge video + chat', self.mergeVideoChat) - if self.mergeVideoChat: - print(f' Layout: {Fore.GREEN}{self.mergeChatLayout}{Style.RESET_ALL}') - self._print_toggle('Cloud upload', self.uploadCloud) - - # Warning messages - if self.deleteFiles: - print(f'\n{Fore.RED}⚠ WARNING: Files will be DELETED after processing{Style.RESET_ALL}') - if not self.uploadCloud: - print(f'{Fore.RED}⚠ CRITICAL: Files will be deleted WITHOUT cloud backup!{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Press CTRL+C to stop and change configuration{Style.RESET_ALL}') - else: - print(f'\n{Fore.GREEN}✓ Files will be preserved locally{Style.RESET_ALL}') - - print(f'\n{Fore.CYAN}{"=" * 60}{Style.RESET_ALL}\n') - - def _print_toggle(self, label: str, value: bool) -> None: - """Helper method to print a configuration toggle in a consistent format.""" - status = f'{Fore.GREEN}Enabled{Style.RESET_ALL}' if value else f'{Fore.RED}Disabled{Style.RESET_ALL}' - print(f'{label}: {status}') - - def run(self) -> None: - """ - Main entry point for the application. - - Initializes environment, validates configuration, creates necessary - directories, and starts the monitoring loop. - """ - # Load environment variables - self._load_environment_variables() - - # Initialize all component modules - self._initialize_components() - - # Validate username - self.stream_monitor.validate_username() - - # Verify dependencies - if not verify_streamlink(): - sys.exit(1) - verify_ffmpeg(self.os_type) - if self.downloadVOD or self.downloadCHAT: - verify_twitch_downloader(self.os_type) - - # Print configuration summary - self._print_configuration_summary() - - # Start monitoring - print(f"Monitoring {Fore.GREEN}{self.username}{Style.RESET_ALL} every {Fore.GREEN}{self.refresh}s{Style.RESET_ALL}") - self.notification_manager.send("TWITCH ARCHIVE STARTED", - f"Monitoring {self.username} every {self.refresh} seconds.") - - # Begin the main monitoring loop - self.loopcheck() - - def _interruptible_sleep(self, seconds: float) -> bool: - """ - Sleep for the specified duration, but check for shutdown periodically. - - Args: - seconds: Number of seconds to sleep - - Returns: - bool: True if sleep completed, False if interrupted by shutdown - """ - start_time = time.time() - while time.time() - start_time < seconds: - if self.shutdown_requested: - return False - time.sleep(min(1.0, seconds - (time.time() - start_time))) - return True - - def _signal_handler(self, signum, frame): - """Handle interrupt signals gracefully.""" - if not self.shutdown_requested: - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}⚠ Shutdown requested. Stopping downloads and finalizing...{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - self.shutdown_requested = True - - # Stop current subprocess if running - if self.recorder: - self.recorder.stop() - - def loopcheck(self) -> None: - """ - Main monitoring loop. - - Continuously checks if the streamer is live, and when they are: - 1. Records the live stream - 2. Downloads the VOD - 3. Downloads and renders chat - 4. Uploads everything to cloud storage (if enabled) - 5. Optionally deletes local files after upload - """ - # Set up signal handlers for graceful shutdown - signal.signal(signal.SIGINT, self._signal_handler) - if hasattr(signal, 'SIGTERM'): - signal.signal(signal.SIGTERM, self._signal_handler) - - while not self.shutdown_requested: - try: - # Check stream status using StreamMonitor - response = self.stream_monitor.check_stream_status() - is_live = response['data']['user']['stream'] - - # Stream is offline - if is_live is None: - print(f'{Fore.CYAN}⏳ {self.username} is offline. Checking again in {self.refresh}s...{Style.RESET_ALL}', end='\r') - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - continue - - # Stream is live but not ready yet - if not is_live.get('title'): - print(f'{Fore.YELLOW}⚠ Stream detected but no title yet. Waiting...{Style.RESET_ALL}') - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - continue - - # Stream is live and ready! - print(f'\n{Fore.GREEN}✓ {self.username} is LIVE!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Title: {is_live["title"]}{Style.RESET_ALL}') - - # Create unique stream identifier - stream_id = f"{is_live['createdAt']} - {self.username} - {is_live['title']}" - - # Parse stream start time - live_date = datetime.strptime( - is_live["createdAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Use CURRENT time for filename - current_time = datetime.now() - filename_base = current_time.strftime('%Y%m%d_%Hh%Mm%Ss') - - # Check if stream was already processed - if self.file_manager.is_stream_processed(stream_id): - print(f'{Fore.YELLOW}⚠ Stream was previously recorded, but it\'s still live!{Style.RESET_ALL}') - print(f'{Fore.GREEN}✓ Starting new recording with timestamp: {filename_base}{Style.RESET_ALL}') - else: - self.file_manager.mark_stream_processed(stream_id) - print(f'{Fore.GREEN}✓ New stream detected - starting recording{Style.RESET_ALL}') - - # Determine file paths - live_raw_path = str(self.file_manager.raw_path / f"{PREFIX_LIVE}{filename_base}.ts") - live_proc_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - live_proc_path = str(self.file_manager.video_path / f"{PREFIX_LIVE}{filename_base}{live_proc_ext}") - - # Ensure unique filenames - live_raw_path = get_unique_filename(live_raw_path) - live_proc_path = get_unique_filename(live_proc_path) - filename_base = os.path.splitext(os.path.basename(live_raw_path))[0].replace(PREFIX_LIVE, "") - - print(f'{Fore.CYAN}Output path: {live_raw_path}{Style.RESET_ALL}') - - # Send notification - self.notification_manager.send(f'🔴 Stream Started - {filename_base}', - f'Title: {is_live["title"]}') - - # Start live chat download if enabled - live_chat_process = None - live_chat_method = None # Track which method was used - chat_json_path = str(self.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") - - if self.downloadLiveCHAT: - vod_id = is_live.get('archiveVideo', {}).get('id') if is_live.get('archiveVideo') else None - stream_url = f"https://twitch.tv/{self.username}" - - live_chat_process, live_chat_method = self.downloader.start_live_chat_download_with_fallback( - vod_id=vod_id, - stream_url=stream_url, - json_path=chat_json_path, - use_chat_downloader_primary=self.use_chat_downloader_primary, - no_chat_downloader_fallback=self.no_chat_downloader_fallback, - verbose=self.verbose - ) - - # Record the live stream - recording_completed = self.recorder.record(is_live, live_raw_path) - - # If shutdown was requested during recording, try to finalize - if self.shutdown_requested: - print(f'{Fore.YELLOW}Attempting to process any recorded content...{Style.RESET_ALL}') - - # Process the raw stream file - self.processor.process_raw_stream(live_raw_path, live_proc_path) - - # Wait for live chat download if it was started - live_chat_downloaded = False - if live_chat_process is not None: - live_chat_downloaded = self.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - - # Render live chat if downloaded successfully - chat_rendered_successfully = False - chat_video_path = None - if live_chat_downloaded: - chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = self.processor.build_chat_output_args() - - # Wait for chat file to be fully accessible (not locked) - print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') - if not self.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): - print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - else: - # Get video duration first (needed for chat conversion and trimming) - ffmpeg_path = get_ffmpeg_executable(self.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') - - # Convert chat format if needed (chat_downloader uses different JSON structure) - render_json_path = chat_json_path - if live_chat_method == 'chat_downloader': - print(f'{Fore.CYAN}Converting chat format for rendering...{Style.RESET_ALL}') - converted_path = chat_json_path.replace('.json', '_converted.json') - if self.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): - render_json_path = converted_path - print(f'{Fore.GREEN}✓ Chat format converted successfully{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Failed to convert chat format{Style.RESET_ALL}') - - chat_rendered_successfully = self.downloader.render_chat( - render_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) - - # Merge video and chat if configured - merged_video_path = None - if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): - merged_video_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{live_proc_ext}") - merge_success = self.processor.merge_video_and_chat( - live_proc_path, - chat_video_path, - merged_video_path, - self.mergeChatLayout - ) - - # Skip VOD/chat download if shutdown was requested or vodTimeout is 0 - vod_response = None - if self.shutdown_requested: - print(f'{Fore.YELLOW}Skipping VOD and chat download due to shutdown request{Style.RESET_ALL}') - elif self.vodTimeout == 0: - print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') - else: - # Try to match stream with VOD (with timeout) - print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {self.vodTimeout}s)...{Style.RESET_ALL}') - vod_found = False - vod_wait_start = time.time() - - while time.time() - vod_wait_start < self.vodTimeout and not self.shutdown_requested: - vod_response = self.stream_monitor.get_latest_vod() - - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - vod_found = True - break - - # Wait before checking again - if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') - if not self._interruptible_sleep(min(10, self.vodTimeout - (time.time() - vod_wait_start))): - break - - if not vod_found: - if self.shutdown_requested: - print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') - else: - print(f'\n{Fore.YELLOW}⚠ VOD not found after {self.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') - vod_response = None - - # Process VOD if found - if not self.shutdown_requested and vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') - - # Save metadata - self.file_manager.save_metadata(current_vod, filename_base) - - # Download VOD - vod_ext = '.mp3' if self.quality == 'audio_only' else '.mp4' - vod_path = str(self.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") - self.downloader.download_vod(current_vod, vod_path) - - # Download and render chat from VOD (if not already done via live chat) - if not live_chat_downloaded: - chat_video_path = str(self.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = self.processor.build_chat_output_args() - - # Get VOD duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(self.os_type) - vod_duration = get_video_duration(vod_path, ffmpeg_path) - - chat_rendered_successfully = self.downloader.download_and_render_chat( - current_vod, - chat_json_path, - chat_video_path, - output_args, - video_duration=vod_duration - ) - - # Merge VOD and chat if configured - if chat_rendered_successfully and self.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): - merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - self.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - self.mergeChatLayout - ) - else: - print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') - - # But still merge VOD with existing chat if configured - if self.mergeVideoChat and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): - merged_vod_path = str(self.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - self.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - self.mergeChatLayout - ) - else: - print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') - - # Clean up raw files if configured - self.file_manager.clean_raw_file(live_raw_path) - - # Upload to cloud if configured - upload_success = self.file_manager.upload_to_cloud( - filename_base, - notification_callback=self.notification_manager.send - ) - - # Delete local files if configured and upload succeeded - if self.deleteFiles and upload_success: - self.file_manager.delete_local_files( - filename_base, - live_raw_path, - live_proc_path, - notification_callback=self.notification_manager.send - ) - - # Done processing this stream - if self.shutdown_requested: - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}✓ Stream processing stopped by user{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - break - else: - print(f'\n{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.GREEN}✓ Stream processing complete{Style.RESET_ALL}') - print(f'{Fore.GREEN}{"=" * 60}{Style.RESET_ALL}\n') - self.notification_manager.send(f'✓ Complete - {filename_base}', - 'Stream processing finished. Resuming monitoring.') - self._interruptible_sleep(self.refresh) - - except KeyboardInterrupt: - if not self.shutdown_requested: - self.shutdown_requested = True - print(f'\n{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}⚠ Interrupted. Cleaning up...{Style.RESET_ALL}') - print(f'{Fore.YELLOW}{"=" * 60}{Style.RESET_ALL}\n') - break - - except Exception as e: - print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.RED}✗ ERROR: {str(e)}{Style.RESET_ALL}') - print(f'{Fore.YELLOW}Waiting {self.refresh} seconds before retrying...{Style.RESET_ALL}') - print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') - self.notification_manager.send('⚠ Error - Recovery', - f'Error: {str(e)}\nRetrying after {self.refresh} seconds.') - - if self.shutdown_requested: - break - self._interruptible_sleep(self.refresh) - - # Final cleanup message - print(f'{Fore.GREEN}✓ Monitoring stopped cleanly{Style.RESET_ALL}') - - def _upload_to_cloud(self, filename_base: str) -> bool: - """ - Upload archived files to cloud storage using rclone. - - Args: - filename_base: Base filename (without prefixes/extensions) - - Returns: - bool: True if upload succeeded or is disabled, False if failed - """ - if not self.uploadCloud: - return True # Consider upload "successful" if disabled - - print(f'\n{Fore.CYAN}Uploading to cloud storage...{Style.RESET_ALL}') - self.send_notification(f'☁ Uploading - {filename_base}', 'Uploading files to cloud storage') - - # Create list of files to upload - bin_path = self._get_bin_path() - upload_list_path = os.path.join(bin_path, 'temp', 'upload.txt') - - # Ensure temp directory exists - os.makedirs(os.path.dirname(upload_list_path), exist_ok=True) - - files_to_upload = [ - f"{PREFIX_LIVE}{filename_base}.ts", - f"{PREFIX_LIVE}{filename_base}.mp4", - f"{PREFIX_LIVE}{filename_base}.mp3", - f"{PREFIX_VOD}{filename_base}.ts", - f"{PREFIX_VOD}{filename_base}.mp4", - f"{PREFIX_VOD}{filename_base}.mp3", - f"{PREFIX_METADATA}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.json", - f"{PREFIX_CHAT}{filename_base}.mp4" - ] - - with open(upload_list_path, 'w') as f: - f.write('\n'.join(files_to_upload)) - - # Run rclone - try: - result = subprocess.call([ - 'rclone', 'copy', - str(pathlib.Path(self.root_path).resolve()), - self.rclone_path, - '--include-from', upload_list_path - ]) - - # Clean up upload list - if os.path.exists(upload_list_path): - os.remove(upload_list_path) - - if result == 0: - print(f'{Fore.GREEN}✓ Upload complete{Style.RESET_ALL}') - self.send_notification(f'✓ Upload Success - {filename_base}', - 'All files uploaded successfully') - return True - else: - print(f'{Fore.RED}✗ Upload failed (exit code: {result}){Style.RESET_ALL}') - print(f'{Fore.YELLOW}Files preserved locally due to upload failure{Style.RESET_ALL}') - self.send_notification(f'✗ Upload Failed - {filename_base}', - f'Upload failed with code {result}. Files preserved locally.') - return False - - except Exception as e: - print(f'{Fore.RED}✗ Upload error: {str(e)}{Style.RESET_ALL}') - return False - - def _delete_local_files(self, filename_base: str, live_raw_path: str, live_proc_path: str) -> None: - """ - Delete local archive files after successful upload. - - Args: - filename_base: Base filename (without prefixes/extensions) - live_raw_path: Path to live raw file - live_proc_path: Path to live processed file - """ - print(f'\n{Fore.RED}{"=" * 60}{Style.RESET_ALL}') - print(f'{Fore.RED}⚠ DELETING LOCAL FILES{Style.RESET_ALL}') - print(f'{Fore.RED}{"=" * 60}{Style.RESET_ALL}\n') - - self.send_notification(f'🗑 Deleting - {filename_base}', - 'Deleting local files after successful upload') - - files_to_delete = [] - - # Live files - if not self.cleanRaw and os.path.exists(live_raw_path): - files_to_delete.append(live_raw_path) - if os.path.exists(live_proc_path): - files_to_delete.append(live_proc_path) - - # VOD files - if self.downloadVOD: - vod_raw = os.path.join(self.raw_path, f"{PREFIX_VOD}{filename_base}.ts") - vod_mp4 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp4") - vod_mp3 = os.path.join(self.video_path, f"{PREFIX_VOD}{filename_base}.mp3") - - if not self.cleanRaw and os.path.exists(vod_raw): - files_to_delete.append(vod_raw) - if os.path.exists(vod_mp4): - files_to_delete.append(vod_mp4) - if os.path.exists(vod_mp3): - files_to_delete.append(vod_mp3) - - # Chat files - if self.downloadCHAT: - chat_json = os.path.join(self.chatJSON_path, f"{PREFIX_CHAT}{filename_base}.json") - chat_mp4 = os.path.join(self.chatMP4_path, f"{PREFIX_CHAT}{filename_base}.mp4") - - if os.path.exists(chat_json): - files_to_delete.append(chat_json) - if os.path.exists(chat_mp4): - files_to_delete.append(chat_mp4) - - # Metadata files - if self.downloadMETADATA: - metadata = os.path.join(self.metadata_path, f"{PREFIX_METADATA}{filename_base}.json") - if os.path.exists(metadata): - files_to_delete.append(metadata) - - # Delete all files - for filepath in files_to_delete: - try: - print(f'{Fore.RED} Deleting: {os.path.basename(filepath)}{Style.RESET_ALL}') - os.remove(filepath) - except Exception as e: - print(f'{Fore.YELLOW} ⚠ Failed to delete {filepath}: {e}{Style.RESET_ALL}') - - print(f'{Fore.RED}\n✓ Cleanup complete{Style.RESET_ALL}') - - -# ============================================================================ -# MULTI-STREAMER MANAGER -# ============================================================================ - -class TwitchArchiveManager: - """ - Manages multiple TwitchArchive instances for monitoring multiple streamers. - """ - - def __init__(self, specific_streamer: Optional[str] = None, verbose: bool = False, - chat_only: bool = False, - use_chat_downloader_primary: bool = False, - use_chat_downloader_fallback: bool = True): - """ - Initialize the manager. - - Args: - specific_streamer: If provided, only monitor this streamer (ignore enabled status) - verbose: Enable verbose debug output - chat_only: Only download chat, skip video recording (test mode) - use_chat_downloader_primary: Use chat_downloader as primary chat source - use_chat_downloader_fallback: Enable chat_downloader fallback - """ - self.config_manager = ConfigManager() - self.specific_streamer = specific_streamer - self.verbose = verbose - self.chat_only = chat_only - self.use_chat_downloader_primary = use_chat_downloader_primary - self.use_chat_downloader_fallback = use_chat_downloader_fallback - self.archivers: Dict[str, TwitchArchive] = {} - self.shutdown_requested = False - self.active_recordings: Dict[str, str] = {} # Track active recordings: {username: stream_id} - - # Setup signal handlers - signal.signal(signal.SIGTERM, self._signal_handler) - signal.signal(signal.SIGINT, self._signal_handler) - - def _signal_handler(self, signum, frame): - """Handle shutdown signals gracefully.""" - print(f'\n{Fore.YELLOW}⚠ Shutdown signal received...{Style.RESET_ALL}') - self.shutdown_requested = True - - # Signal all archivers to shut down - for archiver in self.archivers.values(): - archiver.shutdown_requested = True - - def _get_streamers_to_monitor(self) -> list: - """ - Get list of streamers to monitor. - - Returns: - list: List of streamer usernames to monitor - """ - if self.specific_streamer: - # Monitor only the specified streamer (ignore enabled flag) - return [self.specific_streamer] - else: - # Monitor all enabled streamers - return self.config_manager.get_all_enabled_streamers() - - def _initialize_archiver(self, username: str) -> TwitchArchive: - """ - Initialize a TwitchArchive instance for a streamer. - - Args: - username: Twitch username - - Returns: - TwitchArchive: Initialized archiver instance - """ - config = self.config_manager.load_streamer_config(username) - - # Apply command-line overrides for chat_downloader options - config['useChatDownloaderPrimary'] = self.use_chat_downloader_primary - config['useChatDownloaderFallback'] = self.use_chat_downloader_fallback - - archiver = TwitchArchive(config) - return archiver - - def run(self) -> None: - """ - Main entry point for multi-streamer monitoring. - - Monitors all enabled streamers (or a specific one if provided). - """ - print(f'\n{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}') - print(f'{Fore.CYAN}TWITCH ARCHIVE - Multi-Streamer Mode{Style.RESET_ALL}') - if self.chat_only: - print(f'{Fore.YELLOW}🧪 TEST MODE: Chat-Only (Video Recording Disabled){Style.RESET_ALL}') - print(f'{Fore.CYAN}{"=" * 70}{Style.RESET_ALL}\n') - - # Get streamers to monitor - streamers = self._get_streamers_to_monitor() - - if not streamers: - print(f'{Fore.RED}✗ No streamers configured or enabled{Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Create config files in config/streamers/{Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Or run with -u to create a new config{Style.RESET_ALL}') - sys.exit(1) - - if self.chat_only: - print(f'{Fore.YELLOW}📝 Chat-Only Mode Enabled:{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Verbose logging: ON{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Video recording: DISABLED{Style.RESET_ALL}') - print(f'{Fore.CYAN} • Chat download: ENABLED{Style.RESET_ALL}') - print(f'{Fore.CYAN} • VOD download: DISABLED{Style.RESET_ALL}') - print() - - print(f'{Fore.GREEN}Monitoring {len(streamers)} streamer(s):{Style.RESET_ALL}') - for streamer in streamers: - print(f' • {Fore.CYAN}{streamer}{Style.RESET_ALL}') - print() - - # Initialize archivers for all streamers - for username in streamers: - try: - archiver = self._initialize_archiver(username) - - # Load environment and initialize components - archiver._load_environment_variables() - archiver._initialize_components() - - # Validate username through stream_monitor - archiver.stream_monitor.validate_username() - - self.archivers[username] = archiver - print(f'{Fore.GREEN}✓ Initialized {username}{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.RED}✗ Failed to initialize {username}: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - - if not self.archivers: - print(f'{Fore.RED}✗ No archivers could be initialized{Style.RESET_ALL}') - sys.exit(1) - - # Verify dependencies once (shared across all streamers) - print(f'\n{Fore.CYAN}Verifying dependencies...{Style.RESET_ALL}') - first_archiver = next(iter(self.archivers.values())) - if not verify_streamlink(): - sys.exit(1) - verify_ffmpeg(first_archiver.os_type) - if first_archiver.downloadVOD or first_archiver.downloadCHAT: - verify_twitch_downloader(first_archiver.os_type) - - # Print configuration summary for each streamer - for username, archiver in self.archivers.items(): - archiver._print_configuration_summary() - - print(f'\n{Fore.GREEN}🚀 Starting monitoring loop...{Style.RESET_ALL}\n') - - # Start monitoring loop - self._monitoring_loop() - - def _monitoring_loop(self) -> None: - """ - Main monitoring loop for all streamers. - - Checks each streamer's status and processes streams as needed. - """ - last_check = {} - last_status_print = time.time() - - while not self.shutdown_requested: - current_time = time.time() - - # Print periodic status every 60 seconds - if current_time - last_status_print >= 60: - status_line = " | ".join([f"{username}: checking" for username in self.archivers.keys()]) - print(f'{Fore.CYAN}[Status] {status_line}{Style.RESET_ALL}') - last_status_print = current_time - - for username, archiver in self.archivers.items(): - # Check if enough time has passed since last check for this streamer - if username not in last_check or (current_time - last_check[username]) >= archiver.refresh: - last_check[username] = current_time - - # Check stream status - try: - response = archiver.stream_monitor.check_stream_status() - - # Debug: Print the full response (if verbose) - if self.verbose: - print(f'\n{Fore.MAGENTA}[DEBUG {username}] API Response: {response}{Style.RESET_ALL}') - - stream_data = response['data']['user']['stream'] if response else None - - if self.verbose: - print(f'{Fore.MAGENTA}[DEBUG {username}] Stream data: {stream_data}{Style.RESET_ALL}') - - if stream_data: - # Stream is live - check if it has required basic data (title and start time) - if stream_data.get('title') and stream_data.get('createdAt'): - # Create composite stream ID like single-streamer mode - # This prevents duplicate recordings in the same session - stream_id = f"{stream_data['createdAt']} - {username} - {stream_data.get('title', 'Untitled')}" - - if self.verbose: - # Check if VOD ID is available (for live chat) - if stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id'): - print(f'{Fore.MAGENTA}[DEBUG {username}] VOD ID: {stream_data["archiveVideo"]["id"]}{Style.RESET_ALL}') - else: - print(f'{Fore.MAGENTA}[DEBUG {username}] No VOD ID available (VODs may be disabled){Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[DEBUG {username}] Composite Stream ID: {stream_id}{Style.RESET_ALL}') - - # Check if we're currently recording this stream - currently_recording = username in self.active_recordings and self.active_recordings[username] == stream_id - - if self.verbose: - print(f'{Fore.MAGENTA}[DEBUG {username}] Currently recording: {currently_recording}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[DEBUG {username}] Active recordings: {self.active_recordings}{Style.RESET_ALL}') - - # Record if not currently recording (ignore .log file - always record if live) - if not currently_recording: - print(f'\n{Fore.GREEN}[{username}] Stream detected!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Title: {stream_data.get("title", "No title")}{Style.RESET_ALL}') - print(f'{Fore.CYAN}Started at: {stream_data["createdAt"]}{Style.RESET_ALL}') - - # Warn if VOD ID not available - if not (stream_data.get('archiveVideo') and stream_data['archiveVideo'].get('id')): - print(f'{Fore.YELLOW}⚠ VOD ID not available - live chat download will be skipped{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Stream recording will proceed normally{Style.RESET_ALL}') - - # Mark as currently recording - self.active_recordings[username] = stream_id - - # Process the stream (this blocks until stream ends) - self._process_stream(archiver, stream_data, stream_id) - - # Mark as processed in log (for record keeping) - archiver.file_manager.mark_stream_processed(stream_id) - - # Remove from active recordings - if username in self.active_recordings: - del self.active_recordings[username] - else: - if self.verbose: - print(f'{Fore.CYAN}[{username}] Currently recording this stream, skipping duplicate...{Style.RESET_ALL}') - else: - # Stream is live but not fully initialized yet - print(f'{Fore.YELLOW}[{username}] Stream starting up, waiting for stream data...{Style.RESET_ALL}') - else: - # Not live - if self.verbose: - print(f'{Fore.CYAN}[{username}] Offline - checking again in {archiver.refresh}s{Style.RESET_ALL}', end='\r') - - except Exception as e: - print(f'{Fore.RED}[{username}] Error checking stream: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - - # Sleep briefly before next iteration - time.sleep(1) - - def _process_stream(self, archiver: TwitchArchive, stream_info: Dict[str, Any], stream_id: str) -> None: - """ - Process a detected stream for a specific archiver. - - Args: - archiver: The TwitchArchive instance - stream_info: Stream information from API - stream_id: Unique stream ID - """ - # Store stream data - archiver.current_stream_data = { - 'stream_id': stream_id, - 'title': stream_info['title'], - 'started_at': stream_info['createdAt'] - } - - # Generate timestamp and filename - timestamp = datetime.now(timezone('UTC')).strftime("%Y%m%d_%Hh%Mm%Ss") - filename_base = f"{PREFIX_LIVE}{archiver.username}_{timestamp}" - - # Parse stream start time - live_date = datetime.strptime( - stream_info["createdAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Define paths - raw_extension = '.ts' - proc_extension = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - - live_raw_path = str(archiver.file_manager.raw_path / f"{filename_base}{raw_extension}") - live_proc_path = str(archiver.file_manager.video_path / f"{filename_base}{proc_extension}") - chat_json_path = str(archiver.file_manager.chat_json_path / f"{PREFIX_CHAT}{filename_base}.json") - - # Send notification - if not self.chat_only: - archiver.notification_manager.send( - f"Stream Started - {archiver.username}", - f"Recording: {stream_info['title']}" - ) - - # Start live chat download if enabled (with fallback support) - live_chat_process = None - live_chat_method = 'failed' - if archiver.downloadLiveCHAT: - if self.verbose or self.chat_only: - print(f'\n{Fore.MAGENTA}[VERBOSE] Starting chat download process...{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] downloadLiveCHAT: {archiver.downloadLiveCHAT}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderPrimary: {archiver.downloader.use_chat_downloader_primary}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] useChatDownloaderFallback: {archiver.downloader.use_chat_downloader_fallback}{Style.RESET_ALL}') - - # Get VOD ID if available - live_vod_id = None - if stream_info.get('archiveVideo') and stream_info['archiveVideo'].get('id'): - live_vod_id = stream_info['archiveVideo']['id'] - print(f'{Fore.CYAN}Live VOD ID detected: {live_vod_id}{Style.RESET_ALL}') - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] VOD URL: https://www.twitch.tv/videos/{live_vod_id}{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ No VOD ID available - will use fallback if configured{Style.RESET_ALL}') - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] This happens when streamer has VODs disabled{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] chat_downloader fallback will be used if enabled{Style.RESET_ALL}') - - # Try to start live chat download with fallback - try: - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] Calling start_live_chat_download_with_fallback(){Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Username: {archiver.username}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] VOD ID: {live_vod_id}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Output path: {chat_json_path}{Style.RESET_ALL}') - - live_chat_process, live_chat_method = archiver.downloader.start_live_chat_download_with_fallback( - archiver.username, live_vod_id, chat_json_path - ) - - if self.verbose or self.chat_only: - print(f'{Fore.MAGENTA}[VERBOSE] Chat download method selected: {live_chat_method}{Style.RESET_ALL}') - print(f'{Fore.MAGENTA}[VERBOSE] Process handle: {live_chat_process}{Style.RESET_ALL}') - - # If chat_downloader is selected, start it in background thread now (before video recording) - if live_chat_method == 'chat_downloader' and not self.chat_only: - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') - try: - print(f'{Fore.CYAN}Starting chat_downloader in background (concurrent with video)...{Style.RESET_ALL}') - archiver.downloader.start_chat_downloader_thread( - archiver.username, chat_json_path, - shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, - verbose=self.verbose - ) - except Exception as e: - print(f'{Fore.RED}✗ Failed to start chat thread: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_method = 'failed' - - except Exception as e: - print(f'{Fore.RED}✗ Failed to start live chat download: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_method = 'failed' - - # Record livestream (skip in chat-only mode) - if self.chat_only: - print(f'\n{Fore.YELLOW}🧪 Chat-Only Mode: Skipping video recording{Style.RESET_ALL}') - print(f'{Fore.CYAN}Waiting for chat download to complete...{Style.RESET_ALL}') - - # Start chat download based on method - if live_chat_method == 'chat_downloader': - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Starting chat_downloader in background thread...{Style.RESET_ALL}') - try: - print(f'{Fore.CYAN}Using chat_downloader for live chat...{Style.RESET_ALL}') - archiver.downloader.start_chat_downloader_thread( - archiver.username, chat_json_path, - shutdown_check=lambda: self.shutdown_requested or archiver.shutdown_requested, - stream_monitor=archiver.stream_monitor, - verbose=self.verbose or self.chat_only - ) - # Wait for completion - live_chat_downloaded = archiver.downloader.wait_for_chat_thread() - except Exception as e: - print(f'{Fore.RED}✗ chat_downloader failed: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_downloaded = False - elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: - if self.verbose: - print(f'{Fore.MAGENTA}[VERBOSE] Waiting for TwitchDownloaderCLI process...{Style.RESET_ALL}') - live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - else: - live_chat_downloaded = False - - # Report results - if live_chat_downloaded: - print(f'\n{Fore.GREEN}✓ Chat-Only Test Complete!{Style.RESET_ALL}') - print(f'{Fore.CYAN}Chat saved to: {chat_json_path}{Style.RESET_ALL}') - if os.path.exists(chat_json_path): - file_size = os.path.getsize(chat_json_path) - print(f'{Fore.CYAN}File size: {file_size / 1024:.2f} KB{Style.RESET_ALL}') - else: - print(f'\n{Fore.RED}✗ Chat download failed{Style.RESET_ALL}') - - return # Exit early, don't process video - - # Normal mode: Record livestream - recording_successful = archiver.recorder.record(stream_info, live_raw_path) - - # Check if raw file exists (may exist even after interrupted recording) - if not os.path.exists(live_raw_path): - print(f'{Fore.RED}✗ No recording file found, skipping processing{Style.RESET_ALL}') - - # Still wait for chat if it's downloading - if live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: - print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') - archiver.downloader.wait_for_chat_thread(timeout=30) - elif live_chat_method == 'twitch_downloader' and live_chat_process is not None: - print(f'{Fore.CYAN}Waiting for chat download to finish...{Style.RESET_ALL}') - archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path, timeout=30) - - return - - # Get file size to check if anything was recorded - file_size = os.path.getsize(live_raw_path) - if file_size < 1024: # Less than 1KB means essentially nothing was recorded - print(f'{Fore.RED}✗ Recording file too small ({file_size} bytes), skipping processing{Style.RESET_ALL}') - return - - print(f'{Fore.CYAN}Processing recorded content ({file_size / (1024*1024):.2f} MB)...{Style.RESET_ALL}') - - # Process raw stream - if not archiver.onlyRaw: - archiver.processor.process_raw_stream(live_raw_path, live_proc_path) - - # Wait for live chat download if it was started - live_chat_downloaded = False - chat_rendered_successfully = False - chat_video_path = None - - # Handle different chat download methods - if live_chat_method == 'twitch_downloader' and live_chat_process is not None: - # Wait for TwitchDownloaderCLI process - print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') - live_chat_downloaded = archiver.downloader.wait_for_chat_download(live_chat_process, chat_json_path) - elif live_chat_method == 'chat_downloader' and archiver.downloader.chat_thread is not None: - # Wait for chat_downloader thread - print(f'{Fore.CYAN}Waiting for live chat download to complete...{Style.RESET_ALL}') - try: - live_chat_downloaded = archiver.downloader.wait_for_chat_thread() - if live_chat_downloaded: - print(f'{Fore.GREEN}✓ Chat download thread completed successfully{Style.RESET_ALL}') - else: - print(f'{Fore.YELLOW}⚠ Chat download thread completed with errors or no messages{Style.RESET_ALL}') - except Exception as e: - print(f'{Fore.RED}✗ Error waiting for chat download thread: {e}{Style.RESET_ALL}') - import traceback - traceback.print_exc() - live_chat_downloaded = False - - # Render live chat if downloaded successfully - if live_chat_downloaded: - chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = archiver.processor.build_chat_output_args() - - # Wait for chat file to be fully accessible (not locked) - print(f'{Fore.CYAN}Verifying chat file is ready for rendering...{Style.RESET_ALL}') - if not archiver.downloader.wait_for_file_access(chat_json_path, max_attempts=15, delay=0.5): - print(f'{Fore.RED}✗ Chat file is locked, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - else: - # Get video duration first - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - video_duration = get_video_duration(live_proc_path, ffmpeg_path) - - if video_duration is None: - print(f'{Fore.YELLOW}⚠ Could not detect video duration from {live_proc_path}{Style.RESET_ALL}') - print(f'{Fore.YELLOW} Will use chat message timestamps instead{Style.RESET_ALL}') - else: - print(f'{Fore.CYAN}Video duration for chat rendering: {video_duration}s{Style.RESET_ALL}') - - # Convert chat format if chat_downloader was used - render_json_path = chat_json_path - if live_chat_method == 'chat_downloader': - converted_path = chat_json_path.replace('.json', '_converted.json') - print(f'{Fore.CYAN}Chat downloaded with chat_downloader, converting format...{Style.RESET_ALL}') - if archiver.downloader.convert_chat_downloader_to_twitch_format(chat_json_path, converted_path, video_duration): - render_json_path = converted_path - print(f'{Fore.GREEN}✓ Using converted chat file for rendering{Style.RESET_ALL}') - else: - print(f'{Fore.RED}✗ Format conversion failed, skipping rendering{Style.RESET_ALL}') - chat_rendered_successfully = False - render_json_path = None - - if render_json_path: - chat_rendered_successfully = archiver.downloader.render_chat( - render_json_path, - chat_video_path, - output_args, - video_duration=video_duration - ) - - # Merge video and chat if configured - merged_video_path = None - if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(live_proc_path) and os.path.exists(chat_video_path): - merged_video_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{filename_base}{proc_extension}") - archiver.processor.merge_video_and_chat( - live_proc_path, - chat_video_path, - merged_video_path, - archiver.mergeChatLayout - ) - - # Wait for VOD and download it - vod_response = None - if archiver.vodTimeout == 0: - print(f'{Fore.CYAN}VOD check disabled (vodTimeout=0). Skipping VOD download.{Style.RESET_ALL}') - elif archiver.shutdown_requested: - print(f'{Fore.YELLOW}Skipping VOD download due to shutdown request{Style.RESET_ALL}') - else: - # Try to match stream with VOD (with timeout) - print(f'{Fore.CYAN}Waiting for VOD to become available (timeout: {archiver.vodTimeout}s)...{Style.RESET_ALL}') - vod_found = False - vod_wait_start = time.time() - - while time.time() - vod_wait_start < archiver.vodTimeout: - # Check for shutdown request - if archiver.shutdown_requested: - print(f'\n{Fore.YELLOW}VOD check interrupted by shutdown{Style.RESET_ALL}') - break - - vod_response = archiver.stream_monitor.get_latest_vod() - - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - vod_found = True - break - - # Wait before checking again - if not vod_found: - print(f'{Fore.CYAN}VOD not found yet, waiting...{Style.RESET_ALL}', end='\r') - time.sleep(min(10, archiver.vodTimeout - (time.time() - vod_wait_start))) - - if not vod_found: - print(f'\n{Fore.YELLOW}⚠ VOD not found after {archiver.vodTimeout}s - streamer may have VODs disabled{Style.RESET_ALL}') - print(f'{Fore.CYAN} → Live recording and chat (if enabled) were saved successfully{Style.RESET_ALL}') - vod_response = None - - # Process VOD if found - if vod_response and vod_response['data']['user']['videos']['edges']: - current_vod = vod_response['data']['user']['videos']['edges'][0]['node'] - vod_date = datetime.strptime( - current_vod["recordedAt"], '%Y-%m-%dT%H:%M:%SZ' - ).replace(tzinfo=timezone('UTC')).astimezone(tz=None).replace(tzinfo=None) - - # Check if VOD matches the stream (within 1 minute tolerance) - time_tolerance = timedelta(minutes=1) - if (live_date - time_tolerance) <= vod_date <= (live_date + time_tolerance): - print(f'\n{Fore.GREEN}✓ Found matching VOD{Style.RESET_ALL}') - - # Save metadata - if archiver.downloadMETADATA: - archiver.file_manager.save_metadata(current_vod, filename_base) - - # Download VOD - if archiver.downloadVOD: - vod_ext = '.mp3' if archiver.quality == 'audio_only' else '.mp4' - vod_path = str(archiver.file_manager.video_path / f"{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.downloader.download_vod(current_vod, vod_path) - - # Download and render chat from VOD (if not already done via live chat) - if archiver.downloadCHAT and not live_chat_downloaded: - chat_video_path = str(archiver.file_manager.chat_mp4_path / f"{PREFIX_CHAT}{filename_base}.mp4") - output_args = archiver.processor.build_chat_output_args() - - # Get VOD duration to trim chat accordingly - ffmpeg_path = get_ffmpeg_executable(archiver.os_type) - vod_duration = get_video_duration(vod_path, ffmpeg_path) - - chat_rendered_successfully = archiver.downloader.download_and_render_chat( - current_vod, - chat_json_path, - chat_video_path, - output_args, - video_duration=vod_duration - ) - - # Merge VOD and chat if configured - if chat_rendered_successfully and archiver.mergeVideoChat and os.path.exists(vod_path) and os.path.exists(chat_video_path): - merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - archiver.mergeChatLayout - ) - elif live_chat_downloaded: - print(f'{Fore.CYAN}Chat already downloaded from live stream, skipping VOD chat download{Style.RESET_ALL}') - - # But still merge VOD with existing chat if configured - if archiver.mergeVideoChat and archiver.downloadVOD and os.path.exists(vod_path) and chat_video_path and os.path.exists(chat_video_path): - merged_vod_path = str(archiver.file_manager.video_path / f"{PREFIX_MERGED}{PREFIX_VOD}{filename_base}{vod_ext}") - archiver.processor.merge_video_and_chat( - vod_path, - chat_video_path, - merged_vod_path, - archiver.mergeChatLayout - ) - else: - print(f'{Fore.YELLOW}⚠ No matching VOD found for this stream{Style.RESET_ALL}') - elif archiver.downloadMETADATA: - # Save what metadata we have from the live stream - archiver.file_manager.save_metadata(stream_info, filename_base) - - # Clean up raw file if configured - archiver.file_manager.clean_raw_file(live_raw_path) - - # Upload to cloud if configured - upload_success = archiver.file_manager.upload_to_cloud( - filename_base, - notification_callback=archiver.notification_manager.send - ) - - # Delete files if configured - if archiver.deleteFiles and upload_success: - archiver.file_manager.delete_local_files( - filename_base, - live_raw_path, - live_proc_path, - notification_callback=archiver.notification_manager.send - ) - - # Send completion notification - archiver.notification_manager.send( - f"Stream Archived - {archiver.username}", - f"Completed: {stream_info['title']}" - ) - - -# ============================================================================ -# COMMAND-LINE INTERFACE -# ============================================================================ - -def main(argv: list) -> None: - """ - Main entry point for command-line execution. - - Parses command-line arguments and starts the archive system. - - Args: - argv: Command-line arguments - """ - specific_streamer = None - use_legacy_mode = False - - help_msg = f''' -{Fore.CYAN}{"=" * 70} -TWITCH ARCHIVE - Automated Stream Recording & Archiving -{"=" * 70}{Style.RESET_ALL} - -{Fore.GREEN}USAGE:{Style.RESET_ALL} - python twitch-archive.py [OPTIONS] - -{Fore.GREEN}MODES:{Style.RESET_ALL} - • Multi-Streamer Mode (default): - Monitor all enabled streamers from config/streamers/*.json - - • Single-Streamer Mode: - Use -u to monitor only one streamer - - • Legacy Mode: - Uses config.json if it exists (deprecated) - -{Fore.GREEN}OPTIONS:{Style.RESET_ALL} - -h, --help Display this help information - -u, --username Monitor only this Twitch channel - --verbose Enable verbose debug output - --legacy Force legacy mode (use config.json) - --chat-only Test mode: Only download chat (skip video recording) - Automatically enables verbose logging - --use-chat-downloader-primary Use chat_downloader as primary chat source (for testing) - --no-chat-downloader-fallback Disable chat_downloader fallback - -{Fore.GREEN}LEGACY OPTIONS (when using --legacy):{Style.RESET_ALL} - -q, --quality Stream quality: best/source, high/720p, - medium/480p, low/360p, audio_only - -a, --ttv-lol <0|1> Enable ad-blocking (1) or disable (0) - -v, --vod <0|1> Download VODs after stream ends - -c, --chat <0|1> Download and render chat - -m, --metadata <0|1> Download stream metadata - -r, --upload <0|1> Upload to cloud storage via rclone - -d, --delete <0|1> Delete local files after upload (CAREFUL!) - -n, --notifications <0|1> Send email notifications - -{Fore.YELLOW}TIPS:{Style.RESET_ALL} - • Create config/global.json for default settings - • Create config/streamers/.json for each streamer - • Set enabled: true/false in each streamer config - • Set up API credentials in .env file - -{Fore.CYAN}EXAMPLES:{Style.RESET_ALL} - python twitch-archive.py # Monitor all enabled streamers - python twitch-archive.py -u vinesauce # Monitor only vinesauce - python twitch-archive.py -u hackerling --verbose # Monitor with debug output - python twitch-archive.py -u streamername --chat-only # Test chat download only (no video) - python twitch-archive.py --use-chat-downloader-primary # Test chat_downloader library - python twitch-archive.py --legacy # Use old config.json mode - -{Fore.CYAN}{"=" * 70}{Style.RESET_ALL} - ''' - - try: - opts, args = getopt.getopt( - argv, - "h:u:q:a:v:c:m:r:d:n:", - ["help", "username=", "quality=", "ttv-lol=", "vod=", "chat=", - "metadata=", "upload=", "delete=", "notifications=", "legacy", "verbose", - "chat-only", "use-chat-downloader-primary", "no-chat-downloader-fallback"] - ) - except getopt.GetoptError as e: - print(f'{Fore.RED}Error: {e}{Style.RESET_ALL}\n') - print(help_msg) - sys.exit(2) - - # Check if legacy mode is requested or if config.json exists (fallback) - legacy_config_exists = os.path.exists(os.path.join(os.path.dirname(__file__), 'config.json')) - - # Parse command line args - legacy_overrides = {} - verbose_mode = False - chat_only_mode = False - use_chat_downloader_primary = False - use_chat_downloader_fallback = True # Default to enabled - for opt, arg in opts: - if opt in ('-h', '--help'): - print(help_msg) - sys.exit(0) - elif opt in ("-u", "--username"): - specific_streamer = arg - elif opt == "--verbose": - verbose_mode = True - elif opt == "--chat-only": - chat_only_mode = True - verbose_mode = True # Auto-enable verbose for chat-only mode - elif opt == "--legacy": - use_legacy_mode = True - elif opt == "--use-chat-downloader-primary": - use_chat_downloader_primary = True - elif opt == "--no-chat-downloader-fallback": - use_chat_downloader_fallback = False - if opt in ('-h', '--help'): - print(help_msg) - sys.exit(0) - elif opt in ("-u", "--username"): - specific_streamer = arg - elif opt == "--verbose": - verbose_mode = True - elif opt == "--legacy": - use_legacy_mode = True - # Legacy options (only used in legacy mode) - elif opt in ("-q", "--quality"): - legacy_overrides['quality'] = arg - elif opt in ("-a", "--ttv-lol"): - legacy_overrides['streamlink_ttvlol'] = bool(int(arg)) - elif opt in ("-v", "--vod"): - legacy_overrides['downloadVOD'] = bool(int(arg)) - elif opt in ("-c", "--chat"): - legacy_overrides['downloadCHAT'] = bool(int(arg)) - elif opt in ("-m", "--metadata"): - legacy_overrides['downloadMETADATA'] = bool(int(arg)) - elif opt in ("-r", "--upload"): - legacy_overrides['uploadCloud'] = bool(int(arg)) - elif opt in ("-d", "--delete"): - legacy_overrides['deleteFiles'] = bool(int(arg)) - elif opt in ("-n", "--notifications"): - legacy_overrides['notifications'] = bool(int(arg)) - - # Determine which mode to use - if use_legacy_mode or (legacy_config_exists and not specific_streamer and not os.path.exists('config/global.json')): - # Legacy mode: single streamer using config.json - print(f'{Fore.YELLOW}⚠ Using legacy mode (config.json){Style.RESET_ALL}') - print(f'{Fore.CYAN}→ Consider migrating to new config structure (config/global.json + config/streamers/*.json){Style.RESET_ALL}\n') - - twitch_archive = TwitchArchive() # Loads from config.json - - # Apply command-line overrides - for key, value in legacy_overrides.items(): - setattr(twitch_archive, key, value) - - # Apply chat_downloader options - if hasattr(twitch_archive.downloader, 'use_chat_downloader_primary'): - twitch_archive.downloader.use_chat_downloader_primary = use_chat_downloader_primary - if hasattr(twitch_archive.downloader, 'use_chat_downloader_fallback'): - twitch_archive.downloader.use_chat_downloader_fallback = use_chat_downloader_fallback - - # Start the archive system - twitch_archive.run() - else: - # New multi-streamer mode - manager = TwitchArchiveManager( - specific_streamer=specific_streamer, - verbose=verbose_mode, - chat_only=chat_only_mode, - use_chat_downloader_primary=use_chat_downloader_primary, - use_chat_downloader_fallback=use_chat_downloader_fallback - ) - manager.run() - - -if __name__ == "__main__": - try: - main(sys.argv[1:]) - except KeyboardInterrupt: - # Suppress stack trace for clean exit - print(f'\n{Fore.GREEN}✓ Graceful shutdown complete{Style.RESET_ALL}') - sys.exit(0) \ No newline at end of file