Refactor server event processing and entity updates

Changed positions of notes

Barotrauma/BarotraumaServer/ServerSource/Networking/GameServer.cs

@@ -1133,6 +1133,9 @@ namespace Barotrauma.Networking
 
             Log(ClientLogName(c) + " has reported an error: " + errorStr, ServerLog.MessageType.Error);
             GameAnalyticsManager.AddErrorEventOnce("GameServer.HandleClientError:" + errorStrNoName, GameAnalyticsManager.ErrorSeverity.Error, errorStr);
+            Log(
+                $"Entity event state at client error: pending={EntityEventManager.PendingCreateEventCount}, queued={EntityEventManager.EventCount}, unique={EntityEventManager.UniqueEventCount}, buffered={EntityEventManager.BufferedEventCount}, lastCreated={EntityEventManager.LastCreatedEventID}",
+                ServerLog.MessageType.Error);
 
             try
             {

Barotrauma/BarotraumaServer/ServerSource/Networking/NetEntityEvent/ServerEntityEventManager.cs

@@ -5,12 +5,8 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
-using static Barotrauma.EosInterface.Ownership;
 
 
-// DO NOT TOUCH ANYTHING HERE
-// OR EVERYTHING WILL FAIL
 namespace Barotrauma.Networking
 {
     class ServerEntityEvent : NetEntityEvent
@@ -66,14 +62,42 @@ namespace Barotrauma.Networking
 
         public List<ServerEntityEvent> Events
         {
-            get { return events; }
+            get
+            {
+                FlushPendingCreates();
+                return events;
+            }
         }
 
         public List<ServerEntityEvent> UniqueEvents
         {
-            get { return uniqueEvents; }
+            get
+            {
+                FlushPendingCreates();
+                return uniqueEvents;
+            }
         }
 
+        public int PendingCreateEventCount => pendingCreateQueue.Count;
+        public int EventCount
+        {
+            get
+            {
+                FlushPendingCreates();
+                return events.Count;
+            }
+        }
+        public int UniqueEventCount
+        {
+            get
+            {
+                FlushPendingCreates();
+                return uniqueEvents.Count;
+            }
+        }
+        public int BufferedEventCount => bufferedEvents.Count;
+        public UInt16 LastCreatedEventID => ID;
+
         private class BufferedEvent
         {
             public readonly Client Sender;
@@ -124,11 +148,6 @@ namespace Barotrauma.Networking
 
         private readonly ConcurrentQueue<PendingCreateEvent> pendingCreateQueue;
 
-        private readonly Task createEventTask;
-
-        private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
-        private readonly SemaphoreSlim eventSignal = new SemaphoreSlim(0);
-
         public ServerEntityEventManager(GameServer server)
         {
             events = new List<ServerEntityEvent>();
@@ -138,33 +157,24 @@ namespace Barotrauma.Networking
             pendingCreateQueue = new ConcurrentQueue<PendingCreateEvent>();
             lastWarningTime = -10.0;
             SEM = this;
-
-            createEventTask = Task.Run(() => CreateEventProcessorLoop(cancellationTokenSource.Token));
         }
 
-        private async Task CreateEventProcessorLoop(CancellationToken token)
+        public void FlushPendingCreates()
         {
-            while (!token.IsCancellationRequested)
+            if (GameMain.MainThread != null && Thread.CurrentThread != GameMain.MainThread)
             {
-                try
-                {
-                    await eventSignal.WaitAsync(100, token);
-                    ProcessPendingCreateEvents();
-                }
-                catch (OperationCanceledException)
-                {
-                    break;
-                }
+                throw new InvalidOperationException($"{nameof(ServerEntityEventManager)} pending events must be flushed on the main thread.");
             }
+            ProcessPendingCreateEvents();
         }
 
         private void ProcessPendingCreateEvents()
         {
-            // Dequeue and process all pending events currently in the queue.
-            // Use a lock to synchronize modifications to shared lists / ID.
+            // CreateEntityEvent can be called from parallel update code. The queue keeps
+            // that enqueue path safe, while this method is only called from the main tick
+            // before reading or writing entity-event state.
             while (pendingCreateQueue.TryDequeue(out PendingCreateEvent pending))
             {
-                // The original CreateEvent logic (mostly unchanged) but executed under a lock
                 if (pending == null || pending.Entity == null) { continue; }
 
                 var entity = pending.Entity;
@@ -216,34 +226,18 @@ namespace Barotrauma.Networking
         {
             if (!ValidateEntity(entity)) { return; }
 
-            // enqueue and let background task handle the rest
             pendingCreateQueue.Enqueue(new PendingCreateEvent(entity, extraData));
-            
-            if (eventSignal.CurrentCount == 0)
-            {
-                eventSignal.Release();
-            }
         }
 
         public void Dispose()
         {
-            cancellationTokenSource.Cancel();
-            eventSignal.Release();
-            try
-            {
-                createEventTask?.Wait(2000);
-            }
-            catch (AggregateException) { }
-            finally
-            {
-                cancellationTokenSource.Dispose();
-                eventSignal.Dispose();
-            }
+            ClearPendingCreates();
         }
 
-        // Due to intensive access demend and time it takes to refactor, we use try-catch when facing thread-safety issue to skip to next update :(
         public void Update(List<Client> clients)
         {
+            FlushPendingCreates();
+
             foreach (BufferedEvent bufferedEvent in bufferedEvents)
             {
                 if (bufferedEvent.Character == null || bufferedEvent.Character.IsDead)
@@ -329,14 +323,7 @@ namespace Barotrauma.Networking
                     }
                 }
 
-                try
-                {
-                    lastSentToAnyoneTime = events.ToList().Find(e => e.ID == lastSentToAnyone)?.CreateTime ?? Timing.TotalTime;
-                }
-                catch
-                {
-                    lastSentToAnyoneTime = Timing.TotalTime;
-                }
+                lastSentToAnyoneTime = events.Find(e => e.ID == lastSentToAnyone)?.CreateTime ?? Timing.TotalTime;
 
 
                 if (Timing.TotalTime - lastWarningTime > 5.0 &&
@@ -353,15 +340,7 @@ namespace Barotrauma.Networking
 
                 clients.Where(c => c.NeedsMidRoundSync).ForEach(c => { if (NetIdUtils.IdMoreRecent(lastSentToAll, c.FirstNewEventID)) lastSentToAll = (ushort)(c.FirstNewEventID - 1); });
 
-                ServerEntityEvent firstEventToResend;
-                try
-                {
-                    firstEventToResend = events.Find(e => e.ID == (ushort)(lastSentToAll + 1));
-                }
-                catch
-                {
-                    firstEventToResend = null;
-                }
+                ServerEntityEvent firstEventToResend = events.Find(e => e.ID == (ushort)(lastSentToAll + 1));
                  
                 if (firstEventToResend != null &&
                     GameMain.GameSession.RoundDuration > server.ServerSettings.RoundStartSyncDuration &&
@@ -450,6 +429,8 @@ namespace Barotrauma.Networking
         /// </summary>
         public void Write(in SegmentTableWriter<ServerNetSegment> segmentTable, Client client, IWriteMessage msg, out List<NetEntityEvent> sentEvents)
         {
+            FlushPendingCreates();
+
             List<NetEntityEvent> eventsToSync = GetEventsToSync(client);
 
             if (eventsToSync.Count == 0)
@@ -576,6 +557,8 @@ namespace Barotrauma.Networking
 
         public void InitClientMidRoundSync(Client client)
         {
+            FlushPendingCreates();
+
             //no need for midround syncing if no events have been created,
             //or if the first created unique event is still in the event list
             if (uniqueEvents.Count == 0 || (events.Count > 0 && events[0].ID == uniqueEvents[0].ID))
@@ -693,6 +676,8 @@ namespace Barotrauma.Networking
 
         public void Clear()
         {
+            ClearPendingCreates();
+
             ID = 0;
             events.Clear();
 
@@ -709,5 +694,10 @@ namespace Barotrauma.Networking
                 c.LastSentEntityEventID = 0;
             }
         }
+
+        private void ClearPendingCreates()
+        {
+            while (pendingCreateQueue.TryDequeue(out _)) { }
+        }
     }
 }

Barotrauma/BarotraumaServer/ServerSource/PerformenceMonitor.cs

@@ -2,11 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading.Tasks;
 
 
 namespace Barotrauma
@@ -39,7 +35,27 @@ namespace Barotrauma
         }
         public int ConnectClients
         {
-            get { return GameMain.Server.ConnectedClients.Count; }
+            get { return GameMain.Server?.ConnectedClients.Count ?? 0; }
+        }
+
+        public int PendingEntityEvents
+        {
+            get { return GameMain.Server?.EntityEventManager?.PendingCreateEventCount ?? 0; }
+        }
+
+        public int EntityEvents
+        {
+            get { return GameMain.Server?.EntityEventManager?.EventCount ?? 0; }
+        }
+
+        public int UniqueEntityEvents
+        {
+            get { return GameMain.Server?.EntityEventManager?.UniqueEventCount ?? 0; }
+        }
+
+        public int BufferedEntityEvents
+        {
+            get { return GameMain.Server?.EntityEventManager?.BufferedEventCount ?? 0; }
         }
 
         public double RealTickRate
@@ -166,6 +182,10 @@ namespace Barotrauma
                    $"Character Count: {CharacterCount}\n" +
                    $"Clients Count {ConnectClients}\n " +
                    $"PhysicsBody Count: {PhysicsBodyCount}\n" +
+                   $"Entity Events: {EntityEvents}\n" +
+                   $"Unique Entity Events: {UniqueEntityEvents}\n" +
+                   $"Pending Entity Events: {PendingEntityEvents}\n" +
+                   $"Buffered Entity Events: {BufferedEntityEvents}\n" +
                    $"Tick Rate: {RealTickRate}\n" +
                    $"Min Tick Rate: {TickRateLow}\n" +
                    $"Max Tick Rate: {TickRateHigh}\n" +

Barotrauma/BarotraumaShared/SharedSource/Map/MapEntity.cs

@@ -8,7 +8,6 @@ using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
 using System.Xml.Linq;
-using static OneOf.Types.TrueFalseOrNull;
 
 namespace Barotrauma
 {
@@ -643,7 +642,7 @@ namespace Barotrauma
         /// </summary>
         public static void UpdateAll(float deltaTime, Camera cam, ParallelOptions parallelOptions)
         {
-            Random rand = new Random();
+            mapEntityUpdateTick++;
 #if CLIENT
             var sw = new System.Diagnostics.Stopwatch();
             sw.Start();
@@ -656,55 +655,59 @@ namespace Barotrauma
             List<Gap> gapList = Gap.GapList.ToList();
 
             // This should never break again... right?
+
+            //update gaps in random order, because otherwise in rooms with multiple gaps
+            //the water/air will always tend to flow through the first gap in the list,
+            //which may lead to weird behavior like water draining down only through
+            //one gap in a room even if there are several
             int n = gapList.Count;
             while (n > 1)
             {
                 n--;
-                int k = rand.Next(n + 1);
+                int k = Rand.Int(n + 1);
                 (gapList[n], gapList[k]) = (gapList[k], gapList[n]);
             }
 
             var itemList = Item.ItemList.ToList();
 
-            // First phase: parallel updates that have no order dependencies
-            Parallel.Invoke(parallelOptions,
-                () =>
-                {
-                    Parallel.ForEach(hullList, parallelOptions, hull =>
-                    {
-                        hull.Update(deltaTime, cam);
-                    });
- 
-                },
-                // Structure parallel update
-                () =>
-                {
-                    Parallel.ForEach(structureList, parallelOptions, structure =>
-                    {
-                        structure.Update(deltaTime, cam);
-                    });
-                },
-                () =>
-                //update gaps in random order, because otherwise in rooms with multiple gaps
-                //the water/air will always tend to flow through the first gap in the list,
-                //which may lead to weird behavior like water draining down only through
-                //one gap in a room even if there are several
+            int mapEntityUpdateInterval = Math.Max(MapEntityUpdateInterval, 1);
+            int poweredUpdateInterval = Math.Max(PoweredUpdateInterval, 1);
 
-                // moved waterflow reset here to see if we can reduce at least some time
-                {
-                    // PLEASE WORK
-                    Parallel.ForEach(gapList, parallelOptions, gap =>
+            if (mapEntityUpdateTick % mapEntityUpdateInterval == 0)
+            {
+                float mapEntityDeltaTime = deltaTime * mapEntityUpdateInterval;
+
+                Parallel.Invoke(parallelOptions,
+                    () =>
                     {
-                        gap.ResetWaterFlowThisFrame();
-                        gap.Update(deltaTime, cam);
+                        Parallel.ForEach(hullList, parallelOptions, hull =>
+                        {
+                            hull.Update(mapEntityDeltaTime, cam);
+                        });
+                    },
+                    () =>
+                    {
+                        Parallel.ForEach(structureList, parallelOptions, structure =>
+                        {
+                            structure.Update(mapEntityDeltaTime, cam);
+                        });
                     });
-                },
-                // Powered components update
-                () =>
-                {
-                    Powered.UpdatePower(deltaTime);
-                }
-            );
+            }
+
+            foreach (Gap gap in gapList)
+            {
+                gap.ResetWaterFlowThisFrame();
+            }
+
+            foreach (Gap gap in gapList)
+            {
+                gap.Update(deltaTime, cam);
+            }
+
+            if (mapEntityUpdateTick % poweredUpdateInterval == 0)
+            {
+                Powered.UpdatePower(deltaTime * poweredUpdateInterval);
+            }
 
 #if CLIENT
             // Hull Cheats need to be executed after Hull update
@@ -720,27 +723,42 @@ namespace Barotrauma
             // Item update (Item.Update() is not thread-safe and must be executed on the main thread)
             Item.UpdatePendingConditionUpdates(deltaTime);
 
-            Item lastUpdatedItem = null;
-
-            try
+            if (mapEntityUpdateTick % mapEntityUpdateInterval == 0)
             {
-                foreach (Item item in itemList)
+                float itemDeltaTime = deltaTime * mapEntityUpdateInterval;
+                Item lastUpdatedItem = null;
+
+                try
                 {
-                    lastUpdatedItem = item;
-                    item.Update(deltaTime, cam);
+                    foreach (Item item in itemList)
+                    {
+                        if (LuaCsSetup.Instance.Game.UpdatePriorityItems.Contains(item)) { continue; }
+                        lastUpdatedItem = item;
+                        item.Update(itemDeltaTime, cam);
+                    }
+                }
+                catch (InvalidOperationException e)
+                {
+                    GameAnalyticsManager.AddErrorEventOnce(
+                        "MapEntity.UpdateAll:ItemUpdateInvalidOperation",
+                        GameAnalyticsManager.ErrorSeverity.Critical,
+                        $"Error while updating item {lastUpdatedItem?.Name ?? "null"}: {e.Message}");
+                    throw new InvalidOperationException($"Error while updating item {lastUpdatedItem?.Name ?? "null"}", innerException: e);
                 }
             }
-            catch (InvalidOperationException e)
+
+            foreach (var item in LuaCsSetup.Instance.Game.UpdatePriorityItems)
             {
-                GameAnalyticsManager.AddErrorEventOnce(
-                    "MapEntity.UpdateAll:ItemUpdateInvalidOperation",
-                    GameAnalyticsManager.ErrorSeverity.Critical,
-                    $"Error while updating item {lastUpdatedItem?.Name ?? "null"}: {e.Message}");
-                throw new InvalidOperationException($"Error while updating item {lastUpdatedItem?.Name ?? "null"}", innerException: e);
+                if (item.Removed) { continue; }
+
+                item.Update(deltaTime, cam);
             }
 
-            UpdateAllProjSpecific(deltaTime);
-            Spawner?.Update();
+            if (mapEntityUpdateTick % mapEntityUpdateInterval == 0)
+            {
+                UpdateAllProjSpecific(deltaTime * mapEntityUpdateInterval);
+                Spawner?.Update();
+            }
 
 #if CLIENT
             sw.Stop();

Barotrauma/BarotraumaShared/SharedSource/Screens/GameScreen.cs

@@ -281,7 +281,6 @@ namespace Barotrauma
 #endif
 
             SingleThreadActionStandbySignal.Wait();
-
             try
             {
                 GameMain.World.Step((float)Timing.Step);
@@ -292,8 +291,10 @@ namespace Barotrauma
                 DebugConsole.ThrowError(errorMsg, e);
                 GameAnalyticsManager.AddErrorEventOnce("GameScreen.Update:WorldLockedException" + e.Message, GameAnalyticsManager.ErrorSeverity.Critical, errorMsg);
             }
-
-            SingleThreadActionStandbySignal.Release();
+            finally
+            {
+                SingleThreadActionStandbySignal.Release();
+            }
 
 #if CLIENT
             sw.Stop();

Barotrauma/BarotraumaShared/SharedSource/SingleThreadWorker.cs

@@ -1,9 +1,7 @@
-using Barotrauma.Networking;
-using System;
+using System;
 using System.Collections.Concurrent;
 using System.Threading;
 using System.Threading.Tasks;
-using static Barotrauma.EosInterface.Ownership;
 
 namespace Barotrauma
 {
@@ -15,7 +13,8 @@ namespace Barotrauma
 
         private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
         private readonly SemaphoreSlim actionSignal = new SemaphoreSlim(0);
-        private static Task WorkerTask;
+        private readonly Task workerTask;
+        private bool disposed;
 
         public static readonly SemaphoreSlim SingleThreadActionStandbySignal = new SemaphoreSlim(1);
 
@@ -26,28 +25,35 @@ namespace Barotrauma
         public SingleThreadWorker()
         {
             ActionQueue = new ConcurrentQueue<Action>();
-            WorkerTask = CreateProcessTask(cancellationTokenSource.Token);
+            workerTask = CreateProcessTask(cancellationTokenSource.Token);
         }
 
         public void Dispose()
         {
+            if (disposed) { return; }
+            disposed = true;
             cancellationTokenSource.Cancel();
-            WorkerTask.Wait();
-            WorkerTask.Dispose();
-            Instance = null;
+            try
+            {
+                actionSignal.Release();
+                workerTask.Wait(2);
+            }
+            catch (AggregateException) { }
+            catch (ObjectDisposedException) { }
             cancellationTokenSource.Dispose();
             actionSignal.Dispose();
-            SingleThreadActionStandbySignal.Dispose();
         }
 
         private async Task CreateProcessTask(CancellationToken token)
         {
             while (!token.IsCancellationRequested)
             {
+                bool lockTaken = false;
                 try
                 {
                     await actionSignal.WaitAsync(100, token);
                     SingleThreadActionStandbySignal.Wait(CancellationToken.None);
+                    lockTaken = true;
                     RunActions();
                 }
                 catch (OperationCanceledException)
@@ -56,7 +62,10 @@ namespace Barotrauma
                 }
                 finally
                 {
-                    SingleThreadActionStandbySignal.Release();
+                    if (lockTaken)
+                    {
+                        SingleThreadActionStandbySignal.Release();
+                    }
                 }
             }
         }
@@ -68,6 +77,8 @@ namespace Barotrauma
         /// <param name="action"></param>
         public void AddAction(Action action)
         {
+            if (disposed || action == null) { return; }
+
             // enqueue and let background task handle the rest
             ActionQueue.Enqueue(action);
 
@@ -87,7 +98,7 @@ namespace Barotrauma
             {
                 try
                 {
-                    action.Invoke();
+                    action?.Invoke();
                 }
                 catch (Exception e)
                 {
@@ -96,7 +107,7 @@ namespace Barotrauma
                     Console.ForegroundColor = ConsoleColor.Yellow;
                     Console.WriteLine($"WARNING: Error occurred when running Single Thread Actions." +
                         $"If the server didn't crash or stop responding then this should be fine \n{e}");
-                    Console.ForegroundColor = Console.ForegroundColor;
+                    Console.ForegroundColor = originalForeground;
                 }
             }
         }