Added batched storage cache updates to avoid overloading the network when external storage handlers have lots of changes (for example for ProjectE Transmutation Tables)

src/main/java/com/raoulvdberge/refinedstorage/api/network/INetwork.java

@@ -93,8 +93,14 @@ public interface INetwork {
      *
      * @param stack   the stack
      * @param delta   the delta
+     * @param batched whether the delta can be batched to be sent all at once using {@link #sendBatchedItemStorageDeltaToClient()}
      */
-    void sendItemStorageDeltaToClient(ItemStack stack, int delta);
+    void sendItemStorageDeltaToClient(ItemStack stack, int delta, boolean batched);
+
+    /**
+     * Sends batched item storage deltas, accumulated through {@link #sendItemStorageDeltaToClient(ItemStack, int, boolean)}.
+     */
+    void sendBatchedItemStorageDeltaToClient();
 
     /**
      * Sends a grid update packet with all the fluids to all clients that are watching a grid connected to this network.

src/main/java/com/raoulvdberge/refinedstorage/api/storage/IStorageCache.java

@@ -32,8 +32,9 @@ public interface IStorageCache<T> {
      * @param stack      the stack to add, do NOT modify
      * @param size       the size to add
      * @param rebuilding true if this method is called while rebuilding, false otherwise
+     * @param batched    true if this change needs to be batched
      */
-    void add(@Nonnull T stack, int size, boolean rebuilding);
+    void add(@Nonnull T stack, int size, boolean rebuilding, boolean batched);
 
     /**
      * Removes a stack from the cache.
@@ -43,8 +44,9 @@ public interface IStorageCache<T> {
      *
      * @param stack   the stack to remove, do NOT modify
      * @param size    the size to remove
+     * @param batched true if this change needs to be batched, false otherwise
      */
-    void remove(@Nonnull T stack, int size);
+    void remove(@Nonnull T stack, int size, boolean batched);
 
     /**
      * Adds a listener to be called when this storage cache changes.

src/main/java/com/raoulvdberge/refinedstorage/apiimpl/network/node/externalstorage/StorageItemExternal.java

@@ -35,7 +35,7 @@ public abstract class StorageItemExternal implements IStorage<ItemStack> {
             // If we exceed the cache size, than that means this item is added
             if (i >= cache.size()) {
                 if (!actual.isEmpty()) {
-                    network.getItemStorageCache().add(actual, actual.getCount(), false);
+                    network.getItemStorageCache().add(actual, actual.getCount(), false, true);
                 }
 
                 continue;
@@ -45,29 +45,29 @@ public abstract class StorageItemExternal implements IStorage<ItemStack> {
 
             if (!cached.isEmpty() && actual.isEmpty()) {
                 // If the cached is not empty but the actual is, we remove this item
-                network.getItemStorageCache().remove(cached, cached.getCount());
+                network.getItemStorageCache().remove(cached, cached.getCount(), true);
             } else if (cached.isEmpty() && !actual.isEmpty()) {
                 // If the cached is empty and the actual isn't, we added this item
-                network.getItemStorageCache().add(actual, actual.getCount(), false);
+                network.getItemStorageCache().add(actual, actual.getCount(), false, true);
 
                 network.getCraftingManager().track(actual, actual.getCount());
             } else if (cached.isEmpty() && actual.isEmpty()) {
                 // If they're both empty, nothing happens
             } else if (!API.instance().getComparer().isEqualNoQuantity(cached, actual)) {
                 // If both items mismatch, remove the old and add the new
-                network.getItemStorageCache().remove(cached, cached.getCount());
+                network.getItemStorageCache().remove(cached, cached.getCount(), true);
-                network.getItemStorageCache().add(actual, actual.getCount(), false);
+                network.getItemStorageCache().add(actual, actual.getCount(), false, true);
 
                 network.getCraftingManager().track(actual, actual.getCount());
             } else if (cached.getCount() != actual.getCount()) {
                 int delta = actual.getCount() - cached.getCount();
 
                 if (delta > 0) {
-                    network.getItemStorageCache().add(actual, delta, false);
+                    network.getItemStorageCache().add(actual, delta, false, true);
 
                     network.getCraftingManager().track(actual, delta);
                 } else {
-                    network.getItemStorageCache().remove(actual, Math.abs(delta));
+                    network.getItemStorageCache().remove(actual, Math.abs(delta), true);
                 }
             }
         }
@@ -77,12 +77,14 @@ public abstract class StorageItemExternal implements IStorage<ItemStack> {
         if (cache.size() > newStacks.size()) {
             for (int i = newStacks.size(); i < cache.size(); ++i) {
                 if (cache.get(i) != ItemStack.EMPTY) {
-                    network.getItemStorageCache().remove(cache.get(i), cache.get(i).getCount());
+                    network.getItemStorageCache().remove(cache.get(i), cache.get(i).getCount(), true);
                 }
             }
         }
 
         this.cache = newStacks;
+
+        network.sendBatchedItemStorageDeltaToClient();
     }
 
     @Override

src/main/java/com/raoulvdberge/refinedstorage/apiimpl/storage/StorageCacheFluid.java

@@ -45,7 +45,7 @@ public class StorageCacheFluid implements IStorageCache<FluidStack> {
             }
 
             for (FluidStack stack : storage.getStacks()) {
-                add(stack, stack.amount, true);
+                add(stack, stack.amount, true, false);
             }
         }
 
@@ -55,7 +55,7 @@ public class StorageCacheFluid implements IStorageCache<FluidStack> {
     }
 
     @Override
-    public synchronized void add(@Nonnull FluidStack stack, int size, boolean rebuilding) {
+    public synchronized void add(@Nonnull FluidStack stack, int size, boolean rebuilding, boolean batched) {
         list.add(stack, size);
 
         if (!rebuilding) {
@@ -66,7 +66,7 @@ public class StorageCacheFluid implements IStorageCache<FluidStack> {
     }
 
     @Override
-    public synchronized void remove(@Nonnull FluidStack stack, int size) {
+    public synchronized void remove(@Nonnull FluidStack stack, int size, boolean batched) {
         if (list.remove(stack, size)) {
             network.sendFluidStorageDeltaToClient(stack, -size);
 

src/main/java/com/raoulvdberge/refinedstorage/apiimpl/storage/StorageCacheItem.java

@@ -46,7 +46,7 @@ public class StorageCacheItem implements IStorageCache<ItemStack> {
 
             for (ItemStack stack : storage.getStacks()) {
                 if (!stack.isEmpty()) {
-                    add(stack, stack.getCount(), true);
+                    add(stack, stack.getCount(), true, false);
                 }
             }
         }
@@ -57,20 +57,20 @@ public class StorageCacheItem implements IStorageCache<ItemStack> {
     }
 
     @Override
-    public synchronized void add(@Nonnull ItemStack stack, int size, boolean rebuilding) {
+    public synchronized void add(@Nonnull ItemStack stack, int size, boolean rebuilding, boolean batched) {
         list.add(stack, size);
 
         if (!rebuilding) {
-            network.sendItemStorageDeltaToClient(stack, size);
+            network.sendItemStorageDeltaToClient(stack, size, batched);
 
             listeners.forEach(Runnable::run);
         }
     }
 
     @Override
-    public synchronized void remove(@Nonnull ItemStack stack, int size) {
+    public synchronized void remove(@Nonnull ItemStack stack, int size, boolean batched) {
         if (list.remove(stack, size)) {
-            network.sendItemStorageDeltaToClient(stack, -size);
+            network.sendItemStorageDeltaToClient(stack, -size, batched);
 
             listeners.forEach(Runnable::run);
         }

src/main/java/com/raoulvdberge/refinedstorage/apiimpl/storage/StorageCacheItemPortable.java

@@ -41,7 +41,7 @@ public class StorageCacheItemPortable implements IStorageCache<ItemStack> {
     }
 
     @Override
-    public void add(@Nonnull ItemStack stack, int size, boolean rebuilding) {
+    public void add(@Nonnull ItemStack stack, int size, boolean rebuilding, boolean batched) {
         list.add(stack, size);
 
         if (!rebuilding) {
@@ -52,7 +52,7 @@ public class StorageCacheItemPortable implements IStorageCache<ItemStack> {
     }
 
     @Override
-    public void remove(@Nonnull ItemStack stack, int size) {
+    public void remove(@Nonnull ItemStack stack, int size, boolean batched) {
         if (list.remove(stack, size)) {
             portableGrid.getWatchers().forEach(w -> RS.INSTANCE.network.sendTo(new MessageGridItemDelta(null, stack, -size), (EntityPlayerMP) w));
 

src/main/java/com/raoulvdberge/refinedstorage/apiimpl/storage/StorageDiskItemPortable.java

@@ -66,7 +66,7 @@ public class StorageDiskItemPortable implements IStorageDisk<ItemStack> {
             int inserted = parent.getCacheDelta(storedPre, size, remainder);
 
             if (inserted > 0) {
-                portableGrid.getCache().add(stack, inserted, false);
+                portableGrid.getCache().add(stack, inserted, false, false);
             }
         }
 
@@ -79,7 +79,7 @@ public class StorageDiskItemPortable implements IStorageDisk<ItemStack> {
         ItemStack extracted = parent.extract(stack, size, flags, simulate);
 
         if (!simulate && extracted != null) {
-            portableGrid.getCache().remove(extracted, extracted.getCount());
+            portableGrid.getCache().remove(extracted, extracted.getCount(), false);
         }
 
         return extracted;

src/main/java/com/raoulvdberge/refinedstorage/integration/projecte/StorageItemTransmutationTable.java

@@ -13,6 +13,7 @@ import net.minecraftforge.items.ItemHandlerHelper;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -30,20 +31,22 @@ public class StorageItemTransmutationTable extends StorageItemExternal {
 
     @Override
     public Collection<ItemStack> getStacks() {
-        List<ItemStack> stored = new LinkedList<>();
-
         if (externalStorage.getOwner() != null) {
             IKnowledgeProvider provider = ProjectEAPI.getTransmutationProxy().getKnowledgeProviderFor(externalStorage.getOwner());
 
             // @todo: https://github.com/sinkillerj/ProjectE/issues/1591
             if (!provider.getClass().getName().equals("moze_intel.projecte.impl.TransmutationOffline$1")) {
+                List<ItemStack> stored = new LinkedList<>();
+
                 for (ItemStack knowledge : provider.getKnowledge()) {
                     stored.add(ItemHandlerHelper.copyStackWithSize(knowledge, (int) Math.floor(provider.getEmc() / (double) ProjectEAPI.getEMCProxy().getValue(knowledge))));
                 }
+
+                return stored;
             }
         }
 
-        return stored;
+        return Collections.emptyList();
     }
 
     @Nullable

src/main/java/com/raoulvdberge/refinedstorage/network/MessageGridItemDelta.java

@@ -10,16 +10,24 @@ import net.minecraft.item.ItemStack;
 import net.minecraftforge.fml.common.network.simpleimpl.IMessage;
 import net.minecraftforge.fml.common.network.simpleimpl.IMessageHandler;
 import net.minecraftforge.fml.common.network.simpleimpl.MessageContext;
+import org.apache.commons.lang3.tuple.Pair;
 
 import javax.annotation.Nullable;
+import java.util.LinkedList;
+import java.util.List;
 
 public class MessageGridItemDelta implements IMessage, IMessageHandler<MessageGridItemDelta, IMessage> {
     @Nullable
     private INetwork network;
+
+    private List<Pair<ItemStack, Integer>> deltas;
+    @Nullable
     private ItemStack stack;
     private int delta;
 
-    private GridStackItem clientStack;
+    @Nullable
+    private GridStackItem gridStack;
+    private List<Pair<GridStackItem, Integer>> gridStacks;
 
     public MessageGridItemDelta() {
     }
@@ -30,26 +38,64 @@ public class MessageGridItemDelta implements IMessage, IMessageHandler<MessageGr
         this.delta = delta;
     }
 
+    public MessageGridItemDelta(@Nullable INetwork network, List<Pair<ItemStack, Integer>> deltas) {
+        this.network = network;
+        this.deltas = deltas;
+    }
+
     @Override
     public void fromBytes(ByteBuf buf) {
-        clientStack = new GridStackItem(buf);
+        int size = buf.readInt();
+
+        if (size == 1) {
+            gridStack = new GridStackItem(buf);
             delta = buf.readInt();
+        } else {
+            gridStacks = new LinkedList<>();
+
+            for (int i = 0; i < size; ++i) {
+                gridStacks.add(Pair.of(new GridStackItem(buf), buf.readInt()));
+            }
+        }
     }
 
     @Override
     public void toBytes(ByteBuf buf) {
+        if (stack != null) {
+            buf.writeInt(1);
+
             StackUtils.writeItemStack(buf, stack, network, false);
             buf.writeInt(delta);
+        } else {
+            buf.writeInt(deltas.size());
+
+            for (Pair<ItemStack, Integer> delta : deltas) {
+                StackUtils.writeItemStack(buf, delta.getLeft(), network, false);
+                buf.writeInt(delta.getRight());
+            }
+        }
     }
 
     @Override
     public IMessage onMessage(MessageGridItemDelta message, MessageContext ctx) {
-        Item item = message.clientStack.getStack().getItem();
+        if (message.gridStack != null) {
+            process(message.gridStack, message.delta);
+        } else {
+            message.gridStacks.forEach(p -> process(p.getLeft(), p.getRight()));
+        }
+
+        GuiGrid.markForSorting();
+
+        return null;
+    }
+
+    private void process(GridStackItem gridStack, int delta) {
+        Item item = gridStack.getStack().getItem();
 
         for (GridStackItem stack : GuiGrid.ITEMS.get(item)) {
-            if (stack.equals(message.clientStack)) {
+            if (stack.equals(gridStack)) {
-                if (stack.getStack().getCount() + message.delta == 0) {
+                if (stack.getStack().getCount() + delta == 0) {
-                    if (message.clientStack.isCraftable()) {
+                    if (gridStack.isCraftable()) {
                         stack.setDisplayCraftText(true);
                     } else {
                         GuiGrid.ITEMS.remove(item, stack);
@@ -58,23 +104,18 @@ public class MessageGridItemDelta implements IMessage, IMessageHandler<MessageGr
                     if (stack.doesDisplayCraftText()) {
                         stack.setDisplayCraftText(false);
 
-                        stack.getStack().setCount(message.delta);
+                        stack.getStack().setCount(delta);
                     } else {
-                        stack.getStack().grow(message.delta);
+                        stack.getStack().grow(delta);
                     }
                 }
 
-                GuiGrid.markForSorting();
+                return;
-
-                return null;
             }
         }
 
-        message.clientStack.getStack().setCount(message.delta);
+        gridStack.getStack().setCount(delta);
 
-        GuiGrid.ITEMS.put(item, message.clientStack);
+        GuiGrid.ITEMS.put(item, gridStack);
-        GuiGrid.markForSorting();
-
-        return null;
     }
 }

src/main/java/com/raoulvdberge/refinedstorage/tile/TileController.java

@@ -63,6 +63,7 @@ import net.minecraftforge.common.util.Constants;
 import net.minecraftforge.energy.CapabilityEnergy;
 import net.minecraftforge.fluids.FluidStack;
 import net.minecraftforge.items.ItemHandlerHelper;
+import org.apache.commons.lang3.tuple.Pair;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -132,6 +133,8 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
     private ISecurityManager securityManager = new SecurityManager(this);
 
     private IStorageCache<ItemStack> itemStorage = new StorageCacheItem(this);
+    private List<Pair<ItemStack, Integer>> batchedItemStorageDeltas = new LinkedList<>();
+
     private IStorageCache<FluidStack> fluidStorage = new StorageCacheFluid(this);
 
     private Map<String, IReaderWriterChannel> readerWriterChannels = new HashMap<>();
@@ -298,10 +301,25 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
     }
 
     @Override
-    public void sendItemStorageDeltaToClient(ItemStack stack, int delta) {
+    public void sendBatchedItemStorageDeltaToClient() {
+        if (!batchedItemStorageDeltas.isEmpty()) {
+            world.getMinecraftServer().getPlayerList().getPlayers().stream()
+                .filter(player -> isWatchingGrid(player, GridType.NORMAL, GridType.CRAFTING, GridType.PATTERN))
+                .forEach(player -> RS.INSTANCE.network.sendTo(new MessageGridItemDelta(this, batchedItemStorageDeltas), player));
+
+            batchedItemStorageDeltas.clear();
+        }
+    }
+
+    @Override
+    public void sendItemStorageDeltaToClient(ItemStack stack, int delta, boolean batched) {
+        if (!batched) {
             world.getMinecraftServer().getPlayerList().getPlayers().stream()
                 .filter(player -> isWatchingGrid(player, GridType.NORMAL, GridType.CRAFTING, GridType.PATTERN))
                 .forEach(player -> RS.INSTANCE.network.sendTo(new MessageGridItemDelta(this, stack, delta), player));
+        } else {
+            batchedItemStorageDeltas.add(Pair.of(stack, delta));
+        }
     }
 
     @Override
@@ -439,7 +457,7 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
         }
 
         if (!simulate && inserted - insertedExternally > 0) {
-            itemStorage.add(stack, inserted - insertedExternally, false);
+            itemStorage.add(stack, inserted - insertedExternally, false, false);
         }
 
         return remainder;
@@ -484,7 +502,7 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
         }
 
         if (newStack != null && newStack.getCount() - extractedExternally > 0 && !simulate) {
-            itemStorage.remove(newStack, newStack.getCount() - extractedExternally);
+            itemStorage.remove(newStack, newStack.getCount() - extractedExternally, false);
         }
 
         return newStack;
@@ -526,7 +544,7 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
         }
 
         if (inserted > 0) {
-            fluidStorage.add(stack, inserted, false);
+            fluidStorage.add(stack, inserted, false, false);
         }
 
         return remainder;
@@ -567,7 +585,7 @@ public class TileController extends TileBase implements ITickable, INetwork, IRe
         }
 
         if (newStack != null && !simulate) {
-            fluidStorage.remove(newStack, newStack.amount);
+            fluidStorage.remove(newStack, newStack.amount, false);
         }
 
         return newStack;