diff --git a/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/recipes/MCRecipeManager.java b/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/recipes/MCRecipeManager.java index b8a2500..f3f45ce 100644 --- a/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/recipes/MCRecipeManager.java +++ b/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/recipes/MCRecipeManager.java @@ -16,6 +16,7 @@ import minetweaker.api.recipes.ShapedRecipe; import minetweaker.api.recipes.ShapelessRecipe; import minetweaker.mc1710.item.MCItemStack; import minetweaker.mc1710.util.MineTweakerHacks; +import minetweaker.mc1710.util.ThreadSafeBitSet; import minetweaker.util.IEventHandler; import net.minecraft.entity.player.EntityPlayer; import net.minecraft.inventory.Container; @@ -30,12 +31,11 @@ import net.minecraftforge.oredict.ShapelessOreRecipe; import org.apache.commons.lang3.tuple.Pair; import stanhebben.zenscript.annotations.Optional; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RecursiveAction; +import java.util.concurrent.RecursiveTask; +import java.util.stream.Collectors; import static minetweaker.api.minecraft.MineTweakerMC.getIItemStack; import static minetweaker.api.minecraft.MineTweakerMC.getItemStack; @@ -269,6 +269,8 @@ public final class MCRecipeManager implements IRecipeManager { } public abstract static class ActionBaseRemoveRecipes implements IUndoableAction { + public abstract boolean matches(IRecipe r); + public abstract Set find(); public void removeRecipes(Set toRemove ) { recipes.removeIf(toRemove::contains); } @@ -277,7 +279,7 @@ public final class MCRecipeManager implements IRecipeManager { public static class ActionRemoveRecipesNoIngredients extends ActionBaseRemoveRecipes implements IUndoableAction { // pair of output, nbtMatch private final List> outputs = new ArrayList<>(); - private final Set toRemove = new HashSet<>(); + private Set toRemove = new HashSet<>(); public void addOutput(IIngredient output, @Optional boolean nbtMatch) { outputs.add(Pair.of(output, nbtMatch)); @@ -288,19 +290,25 @@ public final class MCRecipeManager implements IRecipeManager { } @Override - public void apply() { - toRemove.clear(); + public boolean matches(IRecipe r) { + ItemStack recipeOutput = r.getRecipeOutput(); + return recipeOutput != null && matches(getIItemStack(recipeOutput)); + } - for (final IRecipe recipe : recipes) { - final ItemStack recipeOutput = recipe.getRecipeOutput(); - if (recipeOutput != null) { - final IItemStack stack = getIItemStack(recipeOutput); - if (matches(stack)) { - toRemove.add(recipe); - } - } - } - super.removeRecipes(toRemove); + @Override + public Set find() { + return recipes.parallelStream() + .filter(r -> { + ItemStack recipeOutput = r.getRecipeOutput(); + return recipeOutput != null && matches(getIItemStack(recipeOutput)); + }).collect(Collectors.toSet()); + } + + @Override + public void apply() { + toRemove = find(); + + removeRecipes(toRemove); } @Override @@ -354,68 +362,77 @@ public final class MCRecipeManager implements IRecipeManager { this.ingredients = ingredients; this.wildcard = wildcard; } - + @Override - public void apply() { - toRemove.clear(); + public boolean matches(IRecipe recipe) { + if (recipe.getRecipeOutput() == null || !output.matches(new MCItemStack(recipe.getRecipeOutput()))) { + return false; + } + if (recipe instanceof ShapedRecipes) { + return false; + } + if (!(recipe instanceof ShapelessRecipes) && !(recipe instanceof ShapelessOreRecipe)) { + return false; + } + if (ingredients != null) { + if (recipe instanceof ShapelessRecipes) { + ShapelessRecipes srecipe = (ShapelessRecipes) recipe; - outer: - for (IRecipe recipe : recipes) { - if (recipe.getRecipeOutput() == null || !output.matches(new MCItemStack(recipe.getRecipeOutput()))) { - continue; - } - if (recipe instanceof ShapedRecipes) { - continue; - } - if (!(recipe instanceof ShapelessRecipes) && !(recipe instanceof ShapelessOreRecipe)) { - continue; - } - if (ingredients != null) { - if (recipe instanceof ShapelessRecipes) { - ShapelessRecipes srecipe = (ShapelessRecipes) recipe; + if (ingredients.length > srecipe.getRecipeSize()) { + return false; + } else if (!wildcard && ingredients.length < srecipe.getRecipeSize()) { + return false; + } - if (ingredients.length > srecipe.getRecipeSize()) { - continue; - } else if (!wildcard && ingredients.length < srecipe.getRecipeSize()) { - continue; - } - - checkIngredient: - for (IIngredient ingredient : ingredients) { - for (int k = 0; k < srecipe.getRecipeSize(); k++) { - if (matches(srecipe.recipeItems.get(k), ingredient)) { - continue checkIngredient; - } + checkIngredient: + for (IIngredient ingredient : ingredients) { + for (int k = 0; k < srecipe.getRecipeSize(); k++) { + if (MCRecipeManager.matches(srecipe.recipeItems.get(k), ingredient)) { + continue checkIngredient; } - - continue outer; - } - } else if (recipe instanceof ShapelessOreRecipe) { - ShapelessOreRecipe srecipe = (ShapelessOreRecipe) recipe; - ArrayList inputs = srecipe.getInput(); - - if (inputs.size() < ingredients.length) { - continue; - } - if (!wildcard && inputs.size() > ingredients.length) { - continue; } - checkIngredient: - for (IIngredient ingredient : ingredients) { - for (int k = 0; k < srecipe.getRecipeSize(); k++) { - if (matches(inputs.get(k), ingredient)) { - continue checkIngredient; - } + return false; + } + } else if (recipe instanceof ShapelessOreRecipe) { + ShapelessOreRecipe srecipe = (ShapelessOreRecipe) recipe; + ArrayList inputs = srecipe.getInput(); + + if (inputs.size() < ingredients.length) { + return false; + } + if (!wildcard && inputs.size() > ingredients.length) { + return false; + } + + checkIngredient: + for (IIngredient ingredient : ingredients) { + for (int k = 0; k < srecipe.getRecipeSize(); k++) { + if (MCRecipeManager.matches(inputs.get(k), ingredient)) { + continue checkIngredient; } - - continue outer; } + + return false; } } - - toRemove.add(recipe); } + return true; + } + + @Override + public Set find() { + Set toRemove = new HashSet<>(); + for (IRecipe recipe : recipes) { + if (matches(recipe)) + toRemove.add(recipe); + } + return toRemove; + } + + @Override + public void apply() { + toRemove = find(); MineTweakerAPI.logInfo("Removing " + toRemove.size() + " Shapeless recipes."); super.removeRecipes(toRemove); } @@ -455,20 +472,17 @@ public final class MCRecipeManager implements IRecipeManager { public static class ActionRemoveShapedRecipes extends ActionBaseRemoveRecipes implements IUndoableAction { final IIngredient output; final IIngredient[][] ingredients; - final Set toRemove = new HashSet<>(); + final int ingredientsWidth; + final int ingredientsHeight; + Set toRemove = new HashSet<>(); public ActionRemoveShapedRecipes(IIngredient output, IIngredient[][] ingredients) { this.output = output; this.ingredients = ingredients; - } - - @Override - public void apply() { int ingredientsWidth = 0; int ingredientsHeight = 0; - toRemove.clear(); if (ingredients != null) { ingredientsHeight = ingredients.length; @@ -477,43 +491,61 @@ public final class MCRecipeManager implements IRecipeManager { ingredientsWidth = Math.max(ingredientsWidth, ingredient.length); } } + this.ingredientsWidth = ingredientsWidth; + this.ingredientsHeight = ingredientsHeight; + } - outer: - for (final IRecipe recipe : recipes) { - final ItemStack output = recipe.getRecipeOutput(); - if (output == null || !this.output.matches(new MCItemStack(output))) { - continue; + @Override + public boolean matches(IRecipe recipe) { + final ItemStack output = recipe.getRecipeOutput(); + if (output == null || !this.output.matches(new MCItemStack(output))) { + return false; + } + if (!(recipe instanceof ShapedRecipes || recipe instanceof ShapedOreRecipe)) + return false; + + if (ingredients != null) { + boolean ore = recipe instanceof ShapedOreRecipe; + final ShapedRecipes shapedRecipe = !ore ? (ShapedRecipes) recipe : null; + final ShapedOreRecipe shapedOreRecipe = ore ? (ShapedOreRecipe) recipe : null; + + final int recipeWidth = ore ? MineTweakerHacks.getShapedOreRecipeWidth(shapedOreRecipe) : shapedRecipe.recipeWidth; + final int recipeHeight = ore ? shapedOreRecipe.getRecipeSize() / recipeWidth : shapedRecipe.recipeHeight; + + if (ingredientsWidth != recipeWidth || ingredientsHeight != recipeHeight) { + return false; } - if (!(recipe instanceof ShapedRecipes || recipe instanceof ShapedOreRecipe)) - continue; - - if (ingredients != null) { - boolean ore = recipe instanceof ShapedOreRecipe; - final ShapedRecipes shapedRecipe = !ore ? (ShapedRecipes) recipe : null; - final ShapedOreRecipe shapedOreRecipe = ore ? (ShapedOreRecipe) recipe : null; - final int recipeWidth = ore ? MineTweakerHacks.getShapedOreRecipeWidth(shapedOreRecipe) : shapedRecipe.recipeWidth; - final int recipeHeight = ore ? shapedOreRecipe.getRecipeSize() / recipeWidth : shapedRecipe.recipeHeight; - - if (ingredientsWidth != recipeWidth || ingredientsHeight != recipeHeight) { - continue; - } + for (int j = 0; j < ingredientsHeight; j++) { + final IIngredient[] row = ingredients[j]; + for (int k = 0; k < ingredientsWidth; k++) { + final IIngredient ingredient = k > row.length ? null : row[k]; + final Object input = (ore ? shapedOreRecipe.getInput() : shapedRecipe.recipeItems)[j * recipeWidth + k]; - for (int j = 0; j < ingredientsHeight; j++) { - final IIngredient[] row = ingredients[j]; - for (int k = 0; k < ingredientsWidth; k++) { - final IIngredient ingredient = k > row.length ? null : row[k]; - final Object input = (ore ? shapedOreRecipe.getInput() : shapedRecipe.recipeItems)[j * recipeWidth + k]; - - if (!matches(input, ingredient)) { - continue outer; - } + if (!MCRecipeManager.matches(input, ingredient)) { + return false; } } } - // If null ingredient list given, remove all ShapedRecipes with the given output - toRemove.add(recipe); } + // If null ingredient list given, remove all ShapedRecipes with the given output + return true; + } + + @Override + public Set find() { + Set toRemove = new HashSet<>(); + + for (final IRecipe recipe : recipes) { + if (matches(recipe)) + toRemove.add(recipe); + } + return toRemove; + } + + @Override + public void apply() { + toRemove = find(); MineTweakerAPI.logInfo(toRemove.size() + " removed"); super.removeRecipes(toRemove); @@ -627,6 +659,45 @@ public final class MCRecipeManager implements IRecipeManager { super(new ShapelessRecipe(output, ingredients, function, action), output, false); } } + + private static class ActionBatchRemoveRecipe implements IUndoableAction { + private final Set recipes; + + private ActionBatchRemoveRecipe(Set recipes) { + this.recipes = recipes; + } + + @Override + public void apply() { + System.out.println("Removing " + recipes.size() + " recipes"); + MCRecipeManager.recipes.removeIf(recipes::contains); + } + + @Override + public boolean canUndo() { + return true; + } + + @Override + public void undo() { + recipesToAdd.addAll(recipes.parallelStream().map(ActionBaseAddRecipe::new).collect(Collectors.toList())); + } + + @Override + public String describe() { + return "Batch Remove " + recipes.size() + " recipes"; + } + + @Override + public String describeUndo() { + return "Add back " + recipes.size() + "recipes"; + } + + @Override + public Object getOverrideKey() { + return null; + } + } public static void applyAdditionsAndRemovals() { System.out.println("MineTweaker: Applying additions and removals"); @@ -634,7 +705,22 @@ public final class MCRecipeManager implements IRecipeManager { if(recipesToUndo.size() > 0) { recipes.removeIf(recipesToUndo::contains); } - MCRecipeManager.recipesToRemove.forEach(MineTweakerAPI::apply); + int commonPoolParallelism = getCommonPoolParallelism(); + if (commonPoolParallelism <= 1) { + // if thread == 1, then we have parallel overhead but doesn't have parallel speedup. + // never do this + recipesToRemove.forEach(ActionBaseRemoveRecipes::apply); + } else + try { + ThreadSafeBitSet removed = new ThreadSafeBitSet(); + new FindRecipeToRemoveTask(removed, recipesToRemove).fork().get(); + MineTweakerAPI.apply(new ActionBatchRemoveRecipe(removed.toBitSet().stream().mapToObj(recipes::get).collect(Collectors.toSet()))); + } catch (InterruptedException | ExecutionException e) { + System.err.println("MineTweaker: Parallel Removal failed. Falling back to sequential remove"); + e.printStackTrace(); + // try it sequentially + recipesToRemove.forEach(ActionBaseRemoveRecipes::apply); + } MCRecipeManager.recipesToAdd.forEach(MineTweakerAPI::apply); actionRemoveRecipesNoIngredients.clearOutputs(); @@ -643,6 +729,10 @@ public final class MCRecipeManager implements IRecipeManager { recipesToUndo.clear(); } + private static int getCommonPoolParallelism() { + return Integer.getInteger("java.util.concurrent.ForkJoinPool.common.parallelism", Runtime.getRuntime().availableProcessors() - 1); + } + public static class HandleLateAdditionsAndRemovals implements IEventHandler { @Override @@ -658,5 +748,40 @@ public final class MCRecipeManager implements IRecipeManager { return false; } } - + + /** + * A ForkJoinTask to find recipes to remove. Slightly more performant than parallelStream(). + */ + private static class FindRecipeToRemoveTask extends RecursiveAction { + + private final ThreadSafeBitSet removed; + private final List workList; + + public FindRecipeToRemoveTask(ThreadSafeBitSet removed, List workList) { + this.removed = removed; + this.workList = workList; + } + + @Override + protected void compute() { + if (workList.isEmpty()) return; + if (workList.size() < 50) { + for (int i = 0; i < recipes.size(); i++) { + if (removed.get(i)) continue; + IRecipe recipe = recipes.get(i); + for (ActionBaseRemoveRecipes r : workList) { + if (r.matches(recipe)) { + removed.set(i); + break; + } + } + } + } else { + FindRecipeToRemoveTask left = new FindRecipeToRemoveTask(removed, workList.subList(0, workList.size() / 2)); + left.fork(); + new FindRecipeToRemoveTask(removed, workList.subList(workList.size() / 2, workList.size())).fork().join(); + left.join(); + } + } + } } diff --git a/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/util/ThreadSafeBitSet.java b/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/util/ThreadSafeBitSet.java new file mode 100644 index 0000000..8c6465b --- /dev/null +++ b/MineTweaker3-MC1710-Main/src/main/java/minetweaker/mc1710/util/ThreadSafeBitSet.java @@ -0,0 +1,441 @@ +/* + * Copyright 2016-2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package minetweaker.mc1710.util; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.BitSet; +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReference; + +/** + * This is a lock-free, thread-safe version of a {@link java.util.BitSet}.

+ * + * Instead of a long array to hold the bits, this implementation uses an AtomicLongArray, then + * does the appropriate compare-and-swap operations when setting the bits. + * + * @author dkoszewnik + * + */ +public class ThreadSafeBitSet { + + public static final int DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS = 14; + + private final int numLongsPerSegment; + private final int log2SegmentSize; + private final int segmentMask; + private final AtomicReference segments; + + public ThreadSafeBitSet() { + this(DEFAULT_LOG2_SEGMENT_SIZE_IN_BITS); /// 16384 bits, 2048 bytes, 256 longs per segment + } + + public ThreadSafeBitSet(int log2SegmentSizeInBits) { + this(log2SegmentSizeInBits, 0); + } + + public ThreadSafeBitSet(int log2SegmentSizeInBits, int numBitsToPreallocate) { + if(log2SegmentSizeInBits < 6) + throw new IllegalArgumentException("Cannot specify fewer than 64 bits in each segment!"); + + this.log2SegmentSize = log2SegmentSizeInBits; + this.numLongsPerSegment = (1 << (log2SegmentSizeInBits - 6)); + this.segmentMask = numLongsPerSegment - 1; + + long numBitsPerSegment = numLongsPerSegment * 64; + int numSegmentsToPreallocate = numBitsToPreallocate == 0 ? 1 : (int)(((numBitsToPreallocate - 1) / numBitsPerSegment) + 1); + + segments = new AtomicReference(); + segments.set(new ThreadSafeBitSetSegments(numSegmentsToPreallocate, numLongsPerSegment)); + } + + public void set(int position) { + int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment + int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment + int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64) + + AtomicLongArray segment = getSegment(segmentPosition); + + long mask = 1L << bitPosition; + + // Thread safety: we need to loop until we win the race to set the long value. + while(true) { + // determine what the new long value will be after we set the appropriate bit. + long currentLongValue = segment.get(longPosition); + long newLongValue = currentLongValue | mask; + + // if no other thread has modified the value since we read it, we won the race and we are done. + if(segment.compareAndSet(longPosition, currentLongValue, newLongValue)) + break; + } + } + + public void clear(int position) { + int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment + int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment + int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64) + + AtomicLongArray segment = getSegment(segmentPosition); + + long mask = ~(1L << bitPosition); + + // Thread safety: we need to loop until we win the race to set the long value. + while(true) { + // determine what the new long value will be after we set the appropriate bit. + long currentLongValue = segment.get(longPosition); + long newLongValue = currentLongValue & mask; + + // if no other thread has modified the value since we read it, we won the race and we are done. + if(segment.compareAndSet(longPosition, currentLongValue, newLongValue)) + break; + } + } + + public boolean get(int position) { + int segmentPosition = position >>> log2SegmentSize; /// which segment -- div by num bits per segment + int longPosition = (position >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment + int bitPosition = position & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64) + + AtomicLongArray segment = getSegment(segmentPosition); + + long mask = 1L << bitPosition; + + return ((segment.get(longPosition) & mask) != 0); + } + + public long maxSetBit() { + ThreadSafeBitSetSegments segments = this.segments.get(); + + int segmentIdx = segments.numSegments() - 1; + + for(;segmentIdx >= 0; segmentIdx--) { + AtomicLongArray segment = segments.getSegment(segmentIdx); + for(int longIdx=segment.length() - 1; longIdx >= 0; longIdx--) { + long l = segment.get(longIdx); + if(l != 0) + return (segmentIdx << log2SegmentSize) + (longIdx * 64) + (63 - Long.numberOfLeadingZeros(l)); + } + } + + return -1; + } + + public int nextSetBit(int fromIndex) { + if (fromIndex < 0) + throw new IndexOutOfBoundsException("fromIndex < 0: " + fromIndex); + + int segmentPosition = fromIndex >>> log2SegmentSize; /// which segment -- div by num bits per segment + + ThreadSafeBitSetSegments segments = this.segments.get(); + + if(segmentPosition >= segments.numSegments()) + return -1; + + int longPosition = (fromIndex >>> 6) & segmentMask; /// which long in the segment -- remainder of div by num bits per segment + int bitPosition = fromIndex & 0x3F; /// which bit in the long -- remainder of div by num bits in long (64) + AtomicLongArray segment = segments.getSegment(segmentPosition); + + long word = segment.get(longPosition) & (0xffffffffffffffffL << bitPosition); + + while (true) { + if (word != 0) + return (segmentPosition << (log2SegmentSize)) + (longPosition << 6) + Long.numberOfTrailingZeros(word); + if (++longPosition > segmentMask) { + segmentPosition++; + if(segmentPosition >= segments.numSegments()) + return -1; + segment = segments.getSegment(segmentPosition); + longPosition = 0; + } + + word = segment.get(longPosition); + } + } + + + /** + * @return the number of bits which are set in this bit set. + */ + public int cardinality() { + ThreadSafeBitSetSegments segments = this.segments.get(); + + int numSetBits = 0; + + for(int i=0;iother bit set.

+ * + * In other words, return a new bit set, which is a bitwise and with the bitwise not of the other bit set. + * + * @param other the other bit set + * @return the resulting bit set + */ + public ThreadSafeBitSet andNot(ThreadSafeBitSet other) { + if(other.log2SegmentSize != log2SegmentSize) + throw new IllegalArgumentException("Segment sizes must be the same"); + + ThreadSafeBitSetSegments thisSegments = this.segments.get(); + ThreadSafeBitSetSegments otherSegments = other.segments.get(); + ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(thisSegments.numSegments(), numLongsPerSegment); + + for(int i=0;i maxNumSegments) + maxNumSegments = segments[i].numSegments(); + } + + ThreadSafeBitSetSegments newSegments = new ThreadSafeBitSetSegments(maxNumSegments, numLongsPerSegment); + + AtomicLongArray segment[] = new AtomicLongArray[segments.length]; + + for(int i=0;isegmentIndex. If this segment does not yet exist, create it. + * + * @param segmentIndex the segment index + * @return the segment + */ + private AtomicLongArray getSegment(int segmentIndex) { + ThreadSafeBitSetSegments visibleSegments = segments.get(); + + while(visibleSegments.numSegments() <= segmentIndex) { + /// Thread safety: newVisibleSegments contains all of the segments from the currently visible segments, plus extra. + /// all of the segments in the currently visible segments are canonical and will not change. + ThreadSafeBitSetSegments newVisibleSegments = new ThreadSafeBitSetSegments(visibleSegments, segmentIndex + 1, numLongsPerSegment); + + /// because we are using a compareAndSet, if this thread "wins the race" and successfully sets this variable, then the segments + /// which are newly defined in newVisibleSegments become canonical. + if(segments.compareAndSet(visibleSegments, newVisibleSegments)) { + visibleSegments = newVisibleSegments; + } else { + /// If we "lose the race" and are growing the ThreadSafeBitSet segments larger, + /// then we will gather the new canonical sets from the update which we missed on the next iteration of this loop. + /// Newly defined segments in newVisibleSegments will be discarded, they do not get to become canonical. + visibleSegments = segments.get(); + } + } + + return visibleSegments.getSegment(segmentIndex); + } + + private static class ThreadSafeBitSetSegments { + + private final AtomicLongArray segments[]; + + private ThreadSafeBitSetSegments(int numSegments, int segmentLength) { + AtomicLongArray segments[] = new AtomicLongArray[numSegments]; + + for(int i=0;i