diff --git a/Barotrauma/BarotraumaServer/ServerSource/Networking/GameServer.cs b/Barotrauma/BarotraumaServer/ServerSource/Networking/GameServer.cs index 160035294..b593d6b7a 100644 --- a/Barotrauma/BarotraumaServer/ServerSource/Networking/GameServer.cs +++ b/Barotrauma/BarotraumaServer/ServerSource/Networking/GameServer.cs @@ -1133,6 +1133,9 @@ namespace Barotrauma.Networking Log(ClientLogName(c) + " has reported an error: " + errorStr, ServerLog.MessageType.Error); GameAnalyticsManager.AddErrorEventOnce("GameServer.HandleClientError:" + errorStrNoName, GameAnalyticsManager.ErrorSeverity.Error, errorStr); + Log( + $"Entity event state at client error: pending={EntityEventManager.PendingCreateEventCount}, queued={EntityEventManager.EventCount}, unique={EntityEventManager.UniqueEventCount}, buffered={EntityEventManager.BufferedEventCount}, lastCreated={EntityEventManager.LastCreatedEventID}", + ServerLog.MessageType.Error); try { diff --git a/Barotrauma/BarotraumaServer/ServerSource/Networking/NetEntityEvent/ServerEntityEventManager.cs b/Barotrauma/BarotraumaServer/ServerSource/Networking/NetEntityEvent/ServerEntityEventManager.cs index 81e2246ae..fef732234 100644 --- a/Barotrauma/BarotraumaServer/ServerSource/Networking/NetEntityEvent/ServerEntityEventManager.cs +++ b/Barotrauma/BarotraumaServer/ServerSource/Networking/NetEntityEvent/ServerEntityEventManager.cs @@ -5,12 +5,8 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; -using System.Threading.Tasks; -using static Barotrauma.EosInterface.Ownership; -// DO NOT TOUCH ANYTHING HERE -// OR EVERYTHING WILL FAIL namespace Barotrauma.Networking { class ServerEntityEvent : NetEntityEvent @@ -66,14 +62,42 @@ namespace Barotrauma.Networking public List Events { - get { return events; } + get + { + FlushPendingCreates(); + return events; + } } public List UniqueEvents { - get { return uniqueEvents; } + get + { + FlushPendingCreates(); + return uniqueEvents; + } } + public int PendingCreateEventCount => pendingCreateQueue.Count; + public int EventCount + { + get + { + FlushPendingCreates(); + return events.Count; + } + } + public int UniqueEventCount + { + get + { + FlushPendingCreates(); + return uniqueEvents.Count; + } + } + public int BufferedEventCount => bufferedEvents.Count; + public UInt16 LastCreatedEventID => ID; + private class BufferedEvent { public readonly Client Sender; @@ -124,11 +148,6 @@ namespace Barotrauma.Networking private readonly ConcurrentQueue pendingCreateQueue; - private readonly Task createEventTask; - - private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); - private readonly SemaphoreSlim eventSignal = new SemaphoreSlim(0); - public ServerEntityEventManager(GameServer server) { events = new List(); @@ -138,33 +157,24 @@ namespace Barotrauma.Networking pendingCreateQueue = new ConcurrentQueue(); lastWarningTime = -10.0; SEM = this; - - createEventTask = Task.Run(() => CreateEventProcessorLoop(cancellationTokenSource.Token)); } - private async Task CreateEventProcessorLoop(CancellationToken token) + public void FlushPendingCreates() { - while (!token.IsCancellationRequested) + if (GameMain.MainThread != null && Thread.CurrentThread != GameMain.MainThread) { - try - { - await eventSignal.WaitAsync(100, token); - ProcessPendingCreateEvents(); - } - catch (OperationCanceledException) - { - break; - } + throw new InvalidOperationException($"{nameof(ServerEntityEventManager)} pending events must be flushed on the main thread."); } + ProcessPendingCreateEvents(); } private void ProcessPendingCreateEvents() { - // Dequeue and process all pending events currently in the queue. - // Use a lock to synchronize modifications to shared lists / ID. + // CreateEntityEvent can be called from parallel update code. The queue keeps + // that enqueue path safe, while this method is only called from the main tick + // before reading or writing entity-event state. while (pendingCreateQueue.TryDequeue(out PendingCreateEvent pending)) { - // The original CreateEvent logic (mostly unchanged) but executed under a lock if (pending == null || pending.Entity == null) { continue; } var entity = pending.Entity; @@ -216,34 +226,18 @@ namespace Barotrauma.Networking { if (!ValidateEntity(entity)) { return; } - // enqueue and let background task handle the rest pendingCreateQueue.Enqueue(new PendingCreateEvent(entity, extraData)); - - if (eventSignal.CurrentCount == 0) - { - eventSignal.Release(); - } } public void Dispose() { - cancellationTokenSource.Cancel(); - eventSignal.Release(); - try - { - createEventTask?.Wait(2000); - } - catch (AggregateException) { } - finally - { - cancellationTokenSource.Dispose(); - eventSignal.Dispose(); - } + ClearPendingCreates(); } - // Due to intensive access demend and time it takes to refactor, we use try-catch when facing thread-safety issue to skip to next update :( public void Update(List clients) { + FlushPendingCreates(); + foreach (BufferedEvent bufferedEvent in bufferedEvents) { if (bufferedEvent.Character == null || bufferedEvent.Character.IsDead) @@ -329,14 +323,7 @@ namespace Barotrauma.Networking } } - try - { - lastSentToAnyoneTime = events.ToList().Find(e => e.ID == lastSentToAnyone)?.CreateTime ?? Timing.TotalTime; - } - catch - { - lastSentToAnyoneTime = Timing.TotalTime; - } + lastSentToAnyoneTime = events.Find(e => e.ID == lastSentToAnyone)?.CreateTime ?? Timing.TotalTime; if (Timing.TotalTime - lastWarningTime > 5.0 && @@ -353,15 +340,7 @@ namespace Barotrauma.Networking clients.Where(c => c.NeedsMidRoundSync).ForEach(c => { if (NetIdUtils.IdMoreRecent(lastSentToAll, c.FirstNewEventID)) lastSentToAll = (ushort)(c.FirstNewEventID - 1); }); - ServerEntityEvent firstEventToResend; - try - { - firstEventToResend = events.Find(e => e.ID == (ushort)(lastSentToAll + 1)); - } - catch - { - firstEventToResend = null; - } + ServerEntityEvent firstEventToResend = events.Find(e => e.ID == (ushort)(lastSentToAll + 1)); if (firstEventToResend != null && GameMain.GameSession.RoundDuration > server.ServerSettings.RoundStartSyncDuration && @@ -450,6 +429,8 @@ namespace Barotrauma.Networking /// public void Write(in SegmentTableWriter segmentTable, Client client, IWriteMessage msg, out List sentEvents) { + FlushPendingCreates(); + List eventsToSync = GetEventsToSync(client); if (eventsToSync.Count == 0) @@ -576,6 +557,8 @@ namespace Barotrauma.Networking public void InitClientMidRoundSync(Client client) { + FlushPendingCreates(); + //no need for midround syncing if no events have been created, //or if the first created unique event is still in the event list if (uniqueEvents.Count == 0 || (events.Count > 0 && events[0].ID == uniqueEvents[0].ID)) @@ -693,6 +676,8 @@ namespace Barotrauma.Networking public void Clear() { + ClearPendingCreates(); + ID = 0; events.Clear(); @@ -709,5 +694,10 @@ namespace Barotrauma.Networking c.LastSentEntityEventID = 0; } } + + private void ClearPendingCreates() + { + while (pendingCreateQueue.TryDequeue(out _)) { } + } } } diff --git a/Barotrauma/BarotraumaServer/ServerSource/PerformenceMonitor.cs b/Barotrauma/BarotraumaServer/ServerSource/PerformenceMonitor.cs index f67ce80ed..733ba866b 100644 --- a/Barotrauma/BarotraumaServer/ServerSource/PerformenceMonitor.cs +++ b/Barotrauma/BarotraumaServer/ServerSource/PerformenceMonitor.cs @@ -2,11 +2,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Threading.Tasks; namespace Barotrauma @@ -39,7 +35,27 @@ namespace Barotrauma } public int ConnectClients { - get { return GameMain.Server.ConnectedClients.Count; } + get { return GameMain.Server?.ConnectedClients.Count ?? 0; } + } + + public int PendingEntityEvents + { + get { return GameMain.Server?.EntityEventManager?.PendingCreateEventCount ?? 0; } + } + + public int EntityEvents + { + get { return GameMain.Server?.EntityEventManager?.EventCount ?? 0; } + } + + public int UniqueEntityEvents + { + get { return GameMain.Server?.EntityEventManager?.UniqueEventCount ?? 0; } + } + + public int BufferedEntityEvents + { + get { return GameMain.Server?.EntityEventManager?.BufferedEventCount ?? 0; } } public double RealTickRate @@ -166,6 +182,10 @@ namespace Barotrauma $"Character Count: {CharacterCount}\n" + $"Clients Count {ConnectClients}\n " + $"PhysicsBody Count: {PhysicsBodyCount}\n" + + $"Entity Events: {EntityEvents}\n" + + $"Unique Entity Events: {UniqueEntityEvents}\n" + + $"Pending Entity Events: {PendingEntityEvents}\n" + + $"Buffered Entity Events: {BufferedEntityEvents}\n" + $"Tick Rate: {RealTickRate}\n" + $"Min Tick Rate: {TickRateLow}\n" + $"Max Tick Rate: {TickRateHigh}\n" + diff --git a/Barotrauma/BarotraumaShared/SharedSource/Map/MapEntity.cs b/Barotrauma/BarotraumaShared/SharedSource/Map/MapEntity.cs index 21cec0761..d43c8fa2e 100644 --- a/Barotrauma/BarotraumaShared/SharedSource/Map/MapEntity.cs +++ b/Barotrauma/BarotraumaShared/SharedSource/Map/MapEntity.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Reflection; using System.Threading.Tasks; using System.Xml.Linq; -using static OneOf.Types.TrueFalseOrNull; namespace Barotrauma { @@ -643,7 +642,7 @@ namespace Barotrauma /// public static void UpdateAll(float deltaTime, Camera cam, ParallelOptions parallelOptions) { - Random rand = new Random(); + mapEntityUpdateTick++; #if CLIENT var sw = new System.Diagnostics.Stopwatch(); sw.Start(); @@ -665,46 +664,50 @@ namespace Barotrauma while (n > 1) { n--; - int k = rand.Next(n + 1); + int k = Rand.Int(n + 1); (gapList[n], gapList[k]) = (gapList[k], gapList[n]); } var itemList = Item.ItemList.ToList(); - // First phase: parallel updates that have no order dependencies - Parallel.Invoke(parallelOptions, - () => - { - Parallel.ForEach(hullList, parallelOptions, hull => + int mapEntityUpdateInterval = Math.Max(MapEntityUpdateInterval, 1); + int poweredUpdateInterval = Math.Max(PoweredUpdateInterval, 1); + + if (mapEntityUpdateTick % mapEntityUpdateInterval == 0) + { + float mapEntityDeltaTime = deltaTime * mapEntityUpdateInterval; + + Parallel.Invoke(parallelOptions, + () => { - hull.Update(deltaTime, cam); - }); - - }, - // Structure parallel update - () => - { - Parallel.ForEach(structureList, parallelOptions, structure => + Parallel.ForEach(hullList, parallelOptions, hull => + { + hull.Update(mapEntityDeltaTime, cam); + }); + }, + () => { - structure.Update(deltaTime, cam); + Parallel.ForEach(structureList, parallelOptions, structure => + { + structure.Update(mapEntityDeltaTime, cam); + }); }); - }, - () => - // moved waterflow reset here to see if we can reduce at least some time - { - // PLEASE WORK - Parallel.ForEach(gapList, parallelOptions, gap => - { - gap.ResetWaterFlowThisFrame(); - gap.Update(deltaTime, cam); - }); - }, - // Powered components update - () => - { - Powered.UpdatePower(deltaTime); - } - ); + } + + foreach (Gap gap in gapList) + { + gap.ResetWaterFlowThisFrame(); + } + + foreach (Gap gap in gapList) + { + gap.Update(deltaTime, cam); + } + + if (mapEntityUpdateTick % poweredUpdateInterval == 0) + { + Powered.UpdatePower(deltaTime * poweredUpdateInterval); + } #if CLIENT // Hull Cheats need to be executed after Hull update @@ -720,27 +723,42 @@ namespace Barotrauma // Item update (Item.Update() is not thread-safe and must be executed on the main thread) Item.UpdatePendingConditionUpdates(deltaTime); - Item lastUpdatedItem = null; - - try + if (mapEntityUpdateTick % mapEntityUpdateInterval == 0) { - foreach (Item item in itemList) + float itemDeltaTime = deltaTime * mapEntityUpdateInterval; + Item lastUpdatedItem = null; + + try { - lastUpdatedItem = item; - item.Update(deltaTime, cam); + foreach (Item item in itemList) + { + if (LuaCsSetup.Instance.Game.UpdatePriorityItems.Contains(item)) { continue; } + lastUpdatedItem = item; + item.Update(itemDeltaTime, cam); + } + } + catch (InvalidOperationException e) + { + GameAnalyticsManager.AddErrorEventOnce( + "MapEntity.UpdateAll:ItemUpdateInvalidOperation", + GameAnalyticsManager.ErrorSeverity.Critical, + $"Error while updating item {lastUpdatedItem?.Name ?? "null"}: {e.Message}"); + throw new InvalidOperationException($"Error while updating item {lastUpdatedItem?.Name ?? "null"}", innerException: e); } } - catch (InvalidOperationException e) + + foreach (var item in LuaCsSetup.Instance.Game.UpdatePriorityItems) { - GameAnalyticsManager.AddErrorEventOnce( - "MapEntity.UpdateAll:ItemUpdateInvalidOperation", - GameAnalyticsManager.ErrorSeverity.Critical, - $"Error while updating item {lastUpdatedItem?.Name ?? "null"}: {e.Message}"); - throw new InvalidOperationException($"Error while updating item {lastUpdatedItem?.Name ?? "null"}", innerException: e); + if (item.Removed) { continue; } + + item.Update(deltaTime, cam); } - UpdateAllProjSpecific(deltaTime); - Spawner?.Update(); + if (mapEntityUpdateTick % mapEntityUpdateInterval == 0) + { + UpdateAllProjSpecific(deltaTime * mapEntityUpdateInterval); + Spawner?.Update(); + } #if CLIENT sw.Stop(); diff --git a/Barotrauma/BarotraumaShared/SharedSource/Screens/GameScreen.cs b/Barotrauma/BarotraumaShared/SharedSource/Screens/GameScreen.cs index c210eda8d..46d453cb2 100644 --- a/Barotrauma/BarotraumaShared/SharedSource/Screens/GameScreen.cs +++ b/Barotrauma/BarotraumaShared/SharedSource/Screens/GameScreen.cs @@ -281,7 +281,6 @@ namespace Barotrauma #endif SingleThreadActionStandbySignal.Wait(); - try { GameMain.World.Step((float)Timing.Step); @@ -292,8 +291,10 @@ namespace Barotrauma DebugConsole.ThrowError(errorMsg, e); GameAnalyticsManager.AddErrorEventOnce("GameScreen.Update:WorldLockedException" + e.Message, GameAnalyticsManager.ErrorSeverity.Critical, errorMsg); } - - SingleThreadActionStandbySignal.Release(); + finally + { + SingleThreadActionStandbySignal.Release(); + } #if CLIENT sw.Stop(); diff --git a/Barotrauma/BarotraumaShared/SharedSource/SingleThreadWorker.cs b/Barotrauma/BarotraumaShared/SharedSource/SingleThreadWorker.cs index d83ca9769..1b4a72210 100644 --- a/Barotrauma/BarotraumaShared/SharedSource/SingleThreadWorker.cs +++ b/Barotrauma/BarotraumaShared/SharedSource/SingleThreadWorker.cs @@ -1,9 +1,7 @@ -using Barotrauma.Networking; -using System; +using System; using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; -using static Barotrauma.EosInterface.Ownership; namespace Barotrauma { @@ -15,7 +13,8 @@ namespace Barotrauma private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); private readonly SemaphoreSlim actionSignal = new SemaphoreSlim(0); - private static Task WorkerTask; + private readonly Task workerTask; + private bool disposed; public static readonly SemaphoreSlim SingleThreadActionStandbySignal = new SemaphoreSlim(1); @@ -26,28 +25,35 @@ namespace Barotrauma public SingleThreadWorker() { ActionQueue = new ConcurrentQueue(); - WorkerTask = CreateProcessTask(cancellationTokenSource.Token); + workerTask = CreateProcessTask(cancellationTokenSource.Token); } public void Dispose() { + if (disposed) { return; } + disposed = true; cancellationTokenSource.Cancel(); - WorkerTask.Wait(); - WorkerTask.Dispose(); - Instance = null; + try + { + actionSignal.Release(); + workerTask.Wait(2); + } + catch (AggregateException) { } + catch (ObjectDisposedException) { } cancellationTokenSource.Dispose(); actionSignal.Dispose(); - SingleThreadActionStandbySignal.Dispose(); } private async Task CreateProcessTask(CancellationToken token) { while (!token.IsCancellationRequested) { + bool lockTaken = false; try { await actionSignal.WaitAsync(100, token); SingleThreadActionStandbySignal.Wait(CancellationToken.None); + lockTaken = true; RunActions(); } catch (OperationCanceledException) @@ -56,7 +62,10 @@ namespace Barotrauma } finally { - SingleThreadActionStandbySignal.Release(); + if (lockTaken) + { + SingleThreadActionStandbySignal.Release(); + } } } } @@ -68,6 +77,8 @@ namespace Barotrauma /// public void AddAction(Action action) { + if (disposed || action == null) { return; } + // enqueue and let background task handle the rest ActionQueue.Enqueue(action); @@ -96,7 +107,7 @@ namespace Barotrauma Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine($"WARNING: Error occurred when running Single Thread Actions." + $"If the server didn't crash or stop responding then this should be fine \n{e}"); - Console.ForegroundColor = Console.ForegroundColor; + Console.ForegroundColor = originalForeground; } } }