From 3af51ab66b368aac18123a7a6b59d0fe0f751db7 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Wed, 5 Aug 2026 15:23:28 +0200 Subject: [PATCH 1/5] feat(main/api/DMLOptions.java): add CLI option for specifying the sparsity estimator NOTE: This option also enables the sparsity rewrites. This will be separated into an individual option soon. feat(main/hops/EstimationUtils.java): create static method to get the sparsity estimator by a given string identifier refactor(main/hops/rewrite/ProgramRewriter.java): separate the conditioned cases for enabling sparsity mm chain rewrites and transpose mm chain rewrites refactor(main/hops/OptimizerUtils.java): rename flag for enabling transpose mm chain rewrites chore(test/**): adapt to the changes above --- .../java/org/apache/sysds/api/DMLOptions.java | 11 ++++++- .../java/org/apache/sysds/api/DMLScript.java | 3 ++ .../org/apache/sysds/hops/OptimizerUtils.java | 4 +-- .../sysds/hops/estim/EstimationUtils.java | 29 +++++++++++++++++++ .../sysds/hops/rewrite/ProgramRewriter.java | 6 ++-- ...riteMatrixMultChainOptimizationSparse.java | 6 ++-- .../rewrite/RewriteMatrixChainDPTest.java | 6 ++-- .../RewriteMatrixMultChainOptSparseTest.java | 10 ++++--- ...ewriteMatrixMultChainOptTransposeTest.java | 6 ++-- 9 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index 10c41e3d0a8..3bcfe7b5874 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -66,6 +66,7 @@ public class DMLOptions { public int fedStatsCount = 10; // Default federated statistics count public boolean memStats = false; // max memory statistics public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type + public String sparsityEstimator = null; // Sparsity estimator to use for rewrites. public ExecMode execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid public boolean gpu = false; // Whether to use the GPU public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU @@ -94,7 +95,7 @@ public class DMLOptions { public boolean federatedCompilation = false; // Compile federated instructions based on input federation state and privacy constraints. public boolean noFedRuntimeConversion = false; // If activated, no runtime conversion of CP instructions to FED instructions will be performed. public int seed = -1; // The general seed for the execution, if -1 random (system time). - public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations + public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations public final static DMLOptions defaultOptions = new DMLOptions(null); @@ -120,6 +121,7 @@ public String toString() { ", oocLogPath=" + oocLogPath + ", memStats=" + memStats + ", explainType=" + explainType + + ", sparsityEstimator=" + sparsityEstimator + ", execMode=" + execMode + ", gpu=" + gpu + ", forceGPU=" + forceGPU + @@ -219,6 +221,9 @@ else if (lineageType.equalsIgnoreCase("debugger")) else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime, codegen, codegen_recompile]"); } } + if(line.hasOption("sparsityEstimator")) { + dmlOptions.sparsityEstimator = line.getOptionValue("sparsityEstimator"); + } dmlOptions.stats = line.hasOption("stats"); if (dmlOptions.stats){ @@ -437,6 +442,9 @@ private static Options createCLIOptions() { Option explainOpt = OptionBuilder.withArgName("level") .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime' / 'codegen' / 'codegen_recompile'") .hasOptionalArg().create("explain"); + Option sparsityEstimatorOpt = OptionBuilder.withArgName("identifier") + .withDescription("specifies the sparsity estimator; can be 'None'[default] / 'Avg' / 'BitsetMM' / 'DM' / 'LG' / 'MNC' / 'MNC_lim' / 'MNC_ext' / 'RS' / 'Sample' / 'SampleRa' / 'Worst'") + .hasOptionalArg().create("sparsityEstimator"); Option execOpt = OptionBuilder.withArgName("mode") .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'HYBRID' / 'spark'") .hasArg().create("exec"); @@ -501,6 +509,7 @@ private static Options createCLIOptions() { options.addOption(oocLogEventsOpt); options.addOption(memOpt); options.addOption(explainOpt); + options.addOption(sparsityEstimatorOpt); options.addOption(execOpt); options.addOption(gpuOpt); options.addOption(oocOpt); diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index a7a175bb7b6..00d3c3ddd0d 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -117,6 +117,8 @@ public class DMLScript public static boolean FED_WORKER = DMLOptions.defaultOptions.fedWorker; // Set explain type public static ExplainType EXPLAIN = DMLOptions.defaultOptions.explainType; + // Enable sparsity rewrites and set sparsity estimator + public static String SPARSITY_ESTIMATOR = DMLOptions.defaultOptions.sparsityEstimator; // Set filename of dml script public static String DML_FILE_PATH_ANTLR_PARSER = DMLOptions.defaultOptions.filePath; // Set data type to use internally @@ -284,6 +286,7 @@ public static boolean executeScript( String[] args ) OOC_LOG_EVENTS = dmlOptions.oocLogEvents; OOC_LOG_PATH = dmlOptions.oocLogPath; EXPLAIN = dmlOptions.explainType; + SPARSITY_ESTIMATOR = dmlOptions.sparsityEstimator; EXEC_MODE = dmlOptions.execMode; LINEAGE = dmlOptions.lineage; LINEAGE_DEDUP = dmlOptions.lineage_dedup; diff --git a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java index 04850cf8637..a364f394de7 100644 --- a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java @@ -200,10 +200,10 @@ public enum MemoryManager { public static boolean ALLOW_SUM_PRODUCT_REWRITES2 = true; /** - * Enables additional mmchain optimizations. In the future, this might be merged with + * Enables transpose mmchain optimizations. In the future, this might be merged with * ALLOW_SUM_PRODUCT_REWRITES. */ - public static boolean ALLOW_ADVANCED_MMCHAIN_REWRITES = false; + public static boolean ALLOW_TRANSPOSE_MMCHAIN_REWRITES = false; /** * Enables a DPSize inspired algorithm rewrite for MMChain with transposes diff --git a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java index eeca0f115fc..53363f08e41 100644 --- a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java +++ b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java @@ -30,6 +30,35 @@ public abstract class EstimationUtils { + public static SparsityEstimator getSparsityEstimator(String identifier) { + switch (identifier) { + case "Avg": + return new EstimatorBasicAvg(); + case "BitsetMM": + return new EstimatorBitsetMM(); + case "DM": + return new EstimatorDensityMap(); + case "LG": + return new EstimatorLayeredGraph(); + case "MNC": + return new EstimatorMatrixHistogram(); + case "MNC_lim": + return new EstimatorMatrixHistogram(false); + case "MNC_ext": + return new EstimatorMatrixHistogram(true); + case "RS": + return new EstimatorRowWise(); + case "Sample": + return new EstimatorSample(); + case "SampleRa": + return new EstimatorSampleRa(); + case "Worst": + return new EstimatorBasicWorst(); + default: + throw new DMLRuntimeException("Unknown sparsity estimator identifier " + identifier); + } + } + /** * This utility function computes the exact output nnz * of a self matrix product without need to materialize diff --git a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java index efc3de5a655..0855a9c542c 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java @@ -139,9 +139,11 @@ public ProgramRewriter(boolean staticRewrites, boolean dynamicRewrites) if( OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE ) { _dagRuleSet.add( new RewriteMatrixMultChainWithTransOptimization() ); } - if(OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES){ + if(OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES){ _dagRuleSet.add( new RewriteMatrixMultChainOptimizationTranspose() ); //dependency: cse - _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); //dependency: cse + } + if(DMLScript.SPARSITY_ESTIMATOR != null && DMLScript.SPARSITY_ESTIMATOR != "None") { + _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); } if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION ) { _dagRuleSet.add( new RewriteAlgebraicSimplificationDynamic() ); //dependencies: cse diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 80b71a1c902..039c6922b14 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -23,10 +23,12 @@ import java.util.List; import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.hops.Hop; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.estim.MMNode; -import org.apache.sysds.hops.estim.EstimatorBasicAvg; +import org.apache.sysds.hops.estim.SparsityEstimator; +import org.apache.sysds.hops.estim.EstimationUtils; import org.apache.sysds.hops.estim.SparsityEstimator.OpCode; /** @@ -85,7 +87,7 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } //compute cost-optimal chains for increasing chain sizes - EstimatorBasicAvg estim = new EstimatorBasicAvg(); + SparsityEstimator estim = EstimationUtils.getSparsityEstimator(DMLScript.SPARSITY_ESTIMATOR); for( int l = 2; l <= size; l++ ) { // chain length for( int i = 0; i < size - l + 1; i++ ) { int j = i + l - 1; diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index 64af7415f88..a975efe7aa0 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -124,13 +124,13 @@ public void setUp() { private void runTestMatrixChainDP(String testName) { ExecMode platformOld = rtplatform; boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; - boolean newMMchain1 = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; + boolean newMMchain1 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; boolean newMMchain2 = OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE; try { rtplatform = ExecMode.SINGLE_NODE; OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = true; - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = true; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = true; OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = true; TestConfiguration config = getTestConfiguration(testName); @@ -301,7 +301,7 @@ private void runTestMatrixChainDP(String testName) { } } finally { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld; - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = newMMchain1; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = newMMchain1; OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = newMMchain2; rtplatform = platformOld; Recompiler.reinitRecompiler(); diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index d4d676dc0e2..199eefc826a 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -103,7 +103,7 @@ public void testMatrixMultChainOptSparseRewrites() { } private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { - boolean oldFlag1 = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; + boolean oldFlag1 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; boolean oldFlag2 = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES; try { @@ -112,11 +112,13 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", + "-sparsityEstimator", rewrites ? "Avg" : "None", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = rewrites; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = rewrites; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewrites; double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsities[0], 7); double[][] Y = getRandomMatrix(cols, 1, -1, 1, sparsities[1], 3); @@ -164,7 +166,7 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { } } finally { - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = oldFlag1; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag1; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = oldFlag2; Recompiler.reinitRecompiler(); } diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java index 72ae5384298..e062f6f0420 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptTransposeTest.java @@ -93,7 +93,7 @@ public void testMMChainFourNoRewrite() { private void testMMChainWithTransposeOperator(String testname, int numOptTranspositions, int numOriginalTranspositions, boolean rewrites) { - boolean oldFlag = OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES; + boolean oldFlag = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; try { TestConfiguration config = getTestConfiguration(testname); loadTestConfiguration(config); @@ -104,7 +104,7 @@ private void testMMChainWithTransposeOperator(String testname, int numOptTranspo fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = rewrites; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = rewrites; //execute tests runTest(true, false, null, -1); @@ -124,7 +124,7 @@ private void testMMChainWithTransposeOperator(String testname, int numOptTranspo } finally { - OptimizerUtils.ALLOW_ADVANCED_MMCHAIN_REWRITES = oldFlag; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag; Recompiler.reinitRecompiler(); } } From 2db428497115425b04934a5aa24b135ec4fde15d Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Wed, 5 Aug 2026 17:23:41 +0200 Subject: [PATCH 2/5] refactor(main/api/DMLOptions.java): create separate option for enabling sparsity rewrites change sparsity estimator option from being the string identifier to the actual enumeration item refactor(main/hops/estim/EstimationUtils.java): add method to get estimator type from identifier string add method to obtain an estimator object directly chore(**): adapt to the above changes chore(test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java): adapt to separate CLI option for enabling sparsity rewrites --- .../java/org/apache/sysds/api/DMLOptions.java | 16 ++- .../java/org/apache/sysds/api/DMLScript.java | 8 +- .../sysds/hops/estim/EstimationUtils.java | 105 +++++++++++++----- .../sysds/hops/rewrite/ProgramRewriter.java | 2 +- ...riteMatrixMultChainOptimizationSparse.java | 5 +- .../RewriteMatrixMultChainOptSparseTest.java | 10 +- 6 files changed, 108 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index 3bcfe7b5874..f17345b107d 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -35,6 +35,7 @@ import org.apache.commons.cli.PosixParser; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.hops.OptimizerUtils; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.runtime.instructions.fed.FEDInstruction; import org.apache.sysds.runtime.instructions.fed.FEDInstructionUtils; import org.apache.sysds.runtime.lineage.LineageCacheConfig.LineageCachePolicy; @@ -66,7 +67,8 @@ public class DMLOptions { public int fedStatsCount = 10; // Default federated statistics count public boolean memStats = false; // max memory statistics public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type - public String sparsityEstimator = null; // Sparsity estimator to use for rewrites. + public boolean sparsityRewrite = false; // Whether to rewrite operation DAGs based on sparsity information. + public EstimatorType sparsityEstimator = EstimatorType.BASIC_AVG; // Sparsity estimator to use for rewrites. public ExecMode execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid public boolean gpu = false; // Whether to use the GPU public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU @@ -121,6 +123,7 @@ public String toString() { ", oocLogPath=" + oocLogPath + ", memStats=" + memStats + ", explainType=" + explainType + + ", sparsityRewrite=" + sparsityRewrite + ", sparsityEstimator=" + sparsityEstimator + ", execMode=" + execMode + ", gpu=" + gpu + @@ -221,8 +224,11 @@ else if (lineageType.equalsIgnoreCase("debugger")) else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime, codegen, codegen_recompile]"); } } + if(line.hasOption("sparsityRewrite")) { + dmlOptions.sparsityRewrite = true; + } if(line.hasOption("sparsityEstimator")) { - dmlOptions.sparsityEstimator = line.getOptionValue("sparsityEstimator"); + dmlOptions.sparsityEstimator = EstimatorType.get(line.getOptionValue("sparsityEstimator")); } dmlOptions.stats = line.hasOption("stats"); @@ -442,8 +448,11 @@ private static Options createCLIOptions() { Option explainOpt = OptionBuilder.withArgName("level") .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime' / 'codegen' / 'codegen_recompile'") .hasOptionalArg().create("explain"); + Option sparsityRewriteOpt = OptionBuilder + .withDescription("enables rewrites of operation DAGs based on sparsity estimates of intermediates.") + .create("sparsityRewrite"); Option sparsityEstimatorOpt = OptionBuilder.withArgName("identifier") - .withDescription("specifies the sparsity estimator; can be 'None'[default] / 'Avg' / 'BitsetMM' / 'DM' / 'LG' / 'MNC' / 'MNC_lim' / 'MNC_ext' / 'RS' / 'Sample' / 'SampleRa' / 'Worst'") + .withDescription("specifies the sparsity estimator; can be 'Avg'[default] / 'BitsetMM' / 'DM' / 'LG' / 'MNC' / 'MNC_lim' / 'MNC_ext' / 'RS' / 'Sample' / 'SampleRa' / 'Worst'") .hasOptionalArg().create("sparsityEstimator"); Option execOpt = OptionBuilder.withArgName("mode") .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'HYBRID' / 'spark'") @@ -509,6 +518,7 @@ private static Options createCLIOptions() { options.addOption(oocLogEventsOpt); options.addOption(memOpt); options.addOption(explainOpt); + options.addOption(sparsityRewriteOpt); options.addOption(sparsityEstimatorOpt); options.addOption(execOpt); options.addOption(gpuOpt); diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index 00d3c3ddd0d..eaedcfb01a1 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -50,6 +50,7 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.codegen.SpoofCompiler; import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.lops.Lop; import org.apache.sysds.parser.DMLProgram; import org.apache.sysds.parser.DMLTranslator; @@ -117,8 +118,10 @@ public class DMLScript public static boolean FED_WORKER = DMLOptions.defaultOptions.fedWorker; // Set explain type public static ExplainType EXPLAIN = DMLOptions.defaultOptions.explainType; - // Enable sparsity rewrites and set sparsity estimator - public static String SPARSITY_ESTIMATOR = DMLOptions.defaultOptions.sparsityEstimator; + // Enable sparsity rewrites + public static boolean SPARSITY_REWRITE = DMLOptions.defaultOptions.sparsityRewrite; + // Set sparsity estimator + public static EstimatorType SPARSITY_ESTIMATOR = DMLOptions.defaultOptions.sparsityEstimator; // Set filename of dml script public static String DML_FILE_PATH_ANTLR_PARSER = DMLOptions.defaultOptions.filePath; // Set data type to use internally @@ -286,6 +289,7 @@ public static boolean executeScript( String[] args ) OOC_LOG_EVENTS = dmlOptions.oocLogEvents; OOC_LOG_PATH = dmlOptions.oocLogPath; EXPLAIN = dmlOptions.explainType; + SPARSITY_REWRITE = dmlOptions.sparsityRewrite; SPARSITY_ESTIMATOR = dmlOptions.sparsityEstimator; EXEC_MODE = dmlOptions.execMode; LINEAGE = dmlOptions.lineage; diff --git a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java index 53363f08e41..d28217a08d2 100644 --- a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java +++ b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java @@ -30,32 +30,85 @@ public abstract class EstimationUtils { - public static SparsityEstimator getSparsityEstimator(String identifier) { - switch (identifier) { - case "Avg": - return new EstimatorBasicAvg(); - case "BitsetMM": - return new EstimatorBitsetMM(); - case "DM": - return new EstimatorDensityMap(); - case "LG": - return new EstimatorLayeredGraph(); - case "MNC": - return new EstimatorMatrixHistogram(); - case "MNC_lim": - return new EstimatorMatrixHistogram(false); - case "MNC_ext": - return new EstimatorMatrixHistogram(true); - case "RS": - return new EstimatorRowWise(); - case "Sample": - return new EstimatorSample(); - case "SampleRa": - return new EstimatorSampleRa(); - case "Worst": - return new EstimatorBasicWorst(); - default: - throw new DMLRuntimeException("Unknown sparsity estimator identifier " + identifier); + /** + * Enumeration for the sparsity estimators supported + */ + public enum EstimatorType { + BASIC_AVG, + BASIC_WORST, + BITSET_MM, + DM, + LG, + MNC, + MNC_LIM, + MNC_EXT, + RS, + SAMPLE, + SAMPLE_RA; + + /** + * @param identifier the string identifier for a sparsity estimator + * @return the estimator type enumeration item + */ + public static EstimatorType get(String identifier) { + switch(identifier) { + case "Avg": + return BASIC_AVG; + case "Worst": + return BASIC_WORST; + case "BitsetMM": + return BITSET_MM; + case "DM": + return DM; + case "LG": + return LG; + case "MNC": + return MNC; + case "MNC_lim": + return MNC_LIM; + case "MNC_ext": + return MNC_EXT; + case "RS": + return RS; + case "Sample": + return SAMPLE; + case "SampleRa": + return SAMPLE_RA; + default: + throw new DMLRuntimeException("Unknown sparsity estimator identifier: " + identifier); + } + } + + /** + * @return a sparsity estimator object corresponding to this estimator type + */ + public SparsityEstimator getEstimator() { + switch(this) { + case BASIC_AVG: + return new EstimatorBasicAvg(); + case BASIC_WORST: + return new EstimatorBasicWorst(); + case BITSET_MM: + return new EstimatorBitsetMM(); + case DM: + return new EstimatorDensityMap(); + case LG: + return new EstimatorLayeredGraph(); + case MNC: + return new EstimatorMatrixHistogram(); + case MNC_LIM: + return new EstimatorMatrixHistogram(false); + case MNC_EXT: + return new EstimatorMatrixHistogram(true); + case RS: + return new EstimatorRowWise(); + case SAMPLE: + return new EstimatorSample(); + case SAMPLE_RA: + return new EstimatorSampleRa(); + default: + throw new DMLRuntimeException("Unknown sparsity estimator " + this.toString()); + } } } diff --git a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java index 0855a9c542c..4c8db95d940 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java @@ -142,7 +142,7 @@ public ProgramRewriter(boolean staticRewrites, boolean dynamicRewrites) if(OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES){ _dagRuleSet.add( new RewriteMatrixMultChainOptimizationTranspose() ); //dependency: cse } - if(DMLScript.SPARSITY_ESTIMATOR != null && DMLScript.SPARSITY_ESTIMATOR != "None") { + if(DMLScript.SPARSITY_REWRITE) { _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); } if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION ) { diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 039c6922b14..6b5d3ed62c0 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -28,7 +28,6 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.estim.MMNode; import org.apache.sysds.hops.estim.SparsityEstimator; -import org.apache.sysds.hops.estim.EstimationUtils; import org.apache.sysds.hops.estim.SparsityEstimator.OpCode; /** @@ -37,7 +36,7 @@ * * Solution: Classic Dynamic Programming * Approach: Currently, the approach based only on matrix dimensions - * and sparsity estimates using the MNC sketch + * and sparsity estimates using the basic average estimator * Goal: To reduce the number of computations in the run-time * (map-reduce) layer */ @@ -87,7 +86,7 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } //compute cost-optimal chains for increasing chain sizes - SparsityEstimator estim = EstimationUtils.getSparsityEstimator(DMLScript.SPARSITY_ESTIMATOR); + SparsityEstimator estim = DMLScript.SPARSITY_ESTIMATOR.getEstimator(); for( int l = 2; l <= size; l++ ) { // chain length for( int i = 0; i < size - l + 1; i++ ) { int j = i + l - 1; diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 199eefc826a..763b1a93861 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -112,9 +112,13 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-explain", "hops", "-stats", - "-sparsityEstimator", rewrites ? "Avg" : "None", - "-args", input("X"), input("Y"), output("R")}; + if(rewrites) + programArgs = new String[] {"-explain", "hops", "-stats", + "-sparsityRewrite", "-sparsityEstimator", "Avg", + "-args", input("X"), input("Y"), output("R")}; + else + programArgs = new String[] {"-explain", "hops", "-stats", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); From a870e730d34ca081b6625e3b1f79cd3f577e3580 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 6 Aug 2026 15:24:22 +0200 Subject: [PATCH 3/5] feat(main/conf/DMLConfig.java): add the XML options for enabling sparsity rewrites and for selecting the respective sparsity estimator fix(main/api/DMLOptions.java): revert adding the cli options for enabling sparsity rewrites and selecting the sparsity estimator fix(main/api/DMLScript.java): revert adding the cli options for enabling sparsity rewrites and selecting the sparsity estimator chore(main/hops/rewrite/ProgramRewriter.java): adapt to the changes above chore(test/RewriteMatrixMultChainOptSparseTest.java): rewrite the XML config file to enable the sparsity rewrites instead of setting the CLI option --- .../java/org/apache/sysds/api/DMLOptions.java | 21 +--------- .../java/org/apache/sysds/api/DMLScript.java | 7 ---- .../java/org/apache/sysds/conf/DMLConfig.java | 6 +++ .../sysds/hops/estim/EstimationUtils.java | 33 ---------------- .../sysds/hops/rewrite/ProgramRewriter.java | 3 +- ...riteMatrixMultChainOptimizationSparse.java | 7 +++- .../RewriteMatrixMultChainOptSparseTest.java | 39 +++++++++++++++---- 7 files changed, 46 insertions(+), 70 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index f17345b107d..10c41e3d0a8 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -35,7 +35,6 @@ import org.apache.commons.cli.PosixParser; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.hops.OptimizerUtils; -import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.runtime.instructions.fed.FEDInstruction; import org.apache.sysds.runtime.instructions.fed.FEDInstructionUtils; import org.apache.sysds.runtime.lineage.LineageCacheConfig.LineageCachePolicy; @@ -67,8 +66,6 @@ public class DMLOptions { public int fedStatsCount = 10; // Default federated statistics count public boolean memStats = false; // max memory statistics public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type - public boolean sparsityRewrite = false; // Whether to rewrite operation DAGs based on sparsity information. - public EstimatorType sparsityEstimator = EstimatorType.BASIC_AVG; // Sparsity estimator to use for rewrites. public ExecMode execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid public boolean gpu = false; // Whether to use the GPU public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU @@ -97,7 +94,7 @@ public class DMLOptions { public boolean federatedCompilation = false; // Compile federated instructions based on input federation state and privacy constraints. public boolean noFedRuntimeConversion = false; // If activated, no runtime conversion of CP instructions to FED instructions will be performed. public int seed = -1; // The general seed for the execution, if -1 random (system time). - public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations + public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations public final static DMLOptions defaultOptions = new DMLOptions(null); @@ -123,8 +120,6 @@ public String toString() { ", oocLogPath=" + oocLogPath + ", memStats=" + memStats + ", explainType=" + explainType + - ", sparsityRewrite=" + sparsityRewrite + - ", sparsityEstimator=" + sparsityEstimator + ", execMode=" + execMode + ", gpu=" + gpu + ", forceGPU=" + forceGPU + @@ -224,12 +219,6 @@ else if (lineageType.equalsIgnoreCase("debugger")) else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime, codegen, codegen_recompile]"); } } - if(line.hasOption("sparsityRewrite")) { - dmlOptions.sparsityRewrite = true; - } - if(line.hasOption("sparsityEstimator")) { - dmlOptions.sparsityEstimator = EstimatorType.get(line.getOptionValue("sparsityEstimator")); - } dmlOptions.stats = line.hasOption("stats"); if (dmlOptions.stats){ @@ -448,12 +437,6 @@ private static Options createCLIOptions() { Option explainOpt = OptionBuilder.withArgName("level") .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime' / 'codegen' / 'codegen_recompile'") .hasOptionalArg().create("explain"); - Option sparsityRewriteOpt = OptionBuilder - .withDescription("enables rewrites of operation DAGs based on sparsity estimates of intermediates.") - .create("sparsityRewrite"); - Option sparsityEstimatorOpt = OptionBuilder.withArgName("identifier") - .withDescription("specifies the sparsity estimator; can be 'Avg'[default] / 'BitsetMM' / 'DM' / 'LG' / 'MNC' / 'MNC_lim' / 'MNC_ext' / 'RS' / 'Sample' / 'SampleRa' / 'Worst'") - .hasOptionalArg().create("sparsityEstimator"); Option execOpt = OptionBuilder.withArgName("mode") .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'HYBRID' / 'spark'") .hasArg().create("exec"); @@ -518,8 +501,6 @@ private static Options createCLIOptions() { options.addOption(oocLogEventsOpt); options.addOption(memOpt); options.addOption(explainOpt); - options.addOption(sparsityRewriteOpt); - options.addOption(sparsityEstimatorOpt); options.addOption(execOpt); options.addOption(gpuOpt); options.addOption(oocOpt); diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index eaedcfb01a1..a7a175bb7b6 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -50,7 +50,6 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.codegen.SpoofCompiler; import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI; -import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.lops.Lop; import org.apache.sysds.parser.DMLProgram; import org.apache.sysds.parser.DMLTranslator; @@ -118,10 +117,6 @@ public class DMLScript public static boolean FED_WORKER = DMLOptions.defaultOptions.fedWorker; // Set explain type public static ExplainType EXPLAIN = DMLOptions.defaultOptions.explainType; - // Enable sparsity rewrites - public static boolean SPARSITY_REWRITE = DMLOptions.defaultOptions.sparsityRewrite; - // Set sparsity estimator - public static EstimatorType SPARSITY_ESTIMATOR = DMLOptions.defaultOptions.sparsityEstimator; // Set filename of dml script public static String DML_FILE_PATH_ANTLR_PARSER = DMLOptions.defaultOptions.filePath; // Set data type to use internally @@ -289,8 +284,6 @@ public static boolean executeScript( String[] args ) OOC_LOG_EVENTS = dmlOptions.oocLogEvents; OOC_LOG_PATH = dmlOptions.oocLogPath; EXPLAIN = dmlOptions.explainType; - SPARSITY_REWRITE = dmlOptions.sparsityRewrite; - SPARSITY_ESTIMATOR = dmlOptions.sparsityEstimator; EXEC_MODE = dmlOptions.execMode; LINEAGE = dmlOptions.lineage; LINEAGE_DEDUP = dmlOptions.lineage_dedup; diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java b/src/main/java/org/apache/sysds/conf/DMLConfig.java index 3a0829922a5..b08c2864597 100644 --- a/src/main/java/org/apache/sysds/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java @@ -43,6 +43,7 @@ import org.apache.sysds.hops.codegen.SpoofCompiler.CompilerType; import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI; import org.apache.sysds.hops.codegen.SpoofCompiler.PlanSelector; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.hops.fedplanner.FTypes.FederatedPlanner; import org.apache.sysds.lops.Compression; import org.apache.sysds.lops.compile.linearization.IDagLinearizerFactory.DagLinearizer; @@ -97,6 +98,8 @@ public class DMLConfig public static final String NATIVE_BLAS = "sysds.native.blas"; public static final String NATIVE_BLAS_DIR = "sysds.native.blas.directory"; public static final String DAG_LINEARIZATION = "sysds.compile.linearization"; + public static final String SPARSITY_REWRITES = "sysds.rewrites.sparsity.enabled"; // boolean + public static final String SPARSITY_ESTIMATOR = "sysds.rewrites.sparsity.estimator"; // see EstiamtionUtils.EstimatorType public static final String CODEGEN = "sysds.codegen.enabled"; //boolean public static final String CODEGEN_API = "sysds.codegen.api"; // see SpoofCompiler.API public static final String CODEGEN_COMPILER = "sysds.codegen.compiler"; //see SpoofCompiler.CompilerType @@ -188,6 +191,8 @@ public class DMLConfig _defaultVals.put(COMPRESSED_TRANSPOSE, "auto"); _defaultVals.put(COMPRESSED_TRANSFORMENCODE, "false"); _defaultVals.put(DAG_LINEARIZATION, DagLinearizer.DEPTH_FIRST.name()); + _defaultVals.put(SPARSITY_REWRITES, "false"); + _defaultVals.put(SPARSITY_ESTIMATOR, EstimatorType.BASIC_AVG.name()); _defaultVals.put(CODEGEN, "false" ); _defaultVals.put(CODEGEN_API, GeneratorAPI.JAVA.name() ); _defaultVals.put(CODEGEN_COMPILER, CompilerType.AUTO.name() ); @@ -476,6 +481,7 @@ public String getConfigInfo() { COMPRESSED_LINALG, COMPRESSED_LOSSY, COMPRESSED_VALID_COMPRESSIONS, COMPRESSED_OVERLAPPING, COMPRESSED_SAMPLING_RATIO, COMPRESSED_SOFT_REFERENCE_COUNT, COMPRESSED_COCODE, COMPRESSED_TRANSPOSE, COMPRESSED_TRANSFORMENCODE, DAG_LINEARIZATION, + SPARSITY_REWRITES, SPARSITY_ESTIMATOR, CODEGEN, CODEGEN_API, CODEGEN_COMPILER, CODEGEN_OPTIMIZER, CODEGEN_PLANCACHE, CODEGEN_LITERALS, STATS_MAX_WRAP_LEN, LINEAGECACHESPILL, COMPILERASSISTED_RW, BUFFERPOOL_LIMIT, MEMORY_MANAGER, PRINT_GPU_MEMORY_INFO, AVAILABLE_GPUS, SYNCHRONIZE_GPU, EAGER_CUDA_FREE, GPU_RULE_BASED_PLACEMENT, diff --git a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java index d28217a08d2..b0552343152 100644 --- a/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java +++ b/src/main/java/org/apache/sysds/hops/estim/EstimationUtils.java @@ -46,39 +46,6 @@ public enum EstimatorType { SAMPLE, SAMPLE_RA; - /** - * @param identifier the string identifier for a sparsity estimator - * @return the estimator type enumeration item - */ - public static EstimatorType get(String identifier) { - switch(identifier) { - case "Avg": - return BASIC_AVG; - case "Worst": - return BASIC_WORST; - case "BitsetMM": - return BITSET_MM; - case "DM": - return DM; - case "LG": - return LG; - case "MNC": - return MNC; - case "MNC_lim": - return MNC_LIM; - case "MNC_ext": - return MNC_EXT; - case "RS": - return RS; - case "Sample": - return SAMPLE; - case "SampleRa": - return SAMPLE_RA; - default: - throw new DMLRuntimeException("Unknown sparsity estimator identifier: " + identifier); - } - } - /** * @return a sparsity estimator object corresponding to this estimator type */ diff --git a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java index 4c8db95d940..73add6c7af0 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/ProgramRewriter.java @@ -24,6 +24,7 @@ import org.apache.sysds.api.DMLScript; import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.conf.CompilerConfig.ConfigType; import org.apache.sysds.hops.Hop; import org.apache.sysds.hops.OptimizerUtils; @@ -142,7 +143,7 @@ public ProgramRewriter(boolean staticRewrites, boolean dynamicRewrites) if(OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES){ _dagRuleSet.add( new RewriteMatrixMultChainOptimizationTranspose() ); //dependency: cse } - if(DMLScript.SPARSITY_REWRITE) { + if(ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_REWRITES)) { _dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() ); } if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION ) { diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 6b5d3ed62c0..865157c6323 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -23,11 +23,13 @@ import java.util.List; import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.sysds.api.DMLScript; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.Hop; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.estim.MMNode; import org.apache.sysds.hops.estim.SparsityEstimator; +import org.apache.sysds.hops.estim.EstimationUtils.EstimatorType; import org.apache.sysds.hops.estim.SparsityEstimator.OpCode; /** @@ -86,7 +88,8 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, } //compute cost-optimal chains for increasing chain sizes - SparsityEstimator estim = DMLScript.SPARSITY_ESTIMATOR.getEstimator(); + SparsityEstimator estim = EstimatorType.valueOf(ConfigurationManager.getDMLConfig() + .getTextValue(DMLConfig.SPARSITY_ESTIMATOR)).getEstimator(); for( int l = 2; l <= size; l++ ) { // chain length for( int i = 0; i < size - l + 1; i++ ) { int j = i + l - 1; diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 763b1a93861..2c7be326c95 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -23,6 +23,8 @@ import org.apache.log4j.Logger; import org.apache.log4j.spi.LoggingEvent; import org.apache.sysds.common.Opcodes; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; import org.apache.sysds.runtime.matrix.data.MatrixValue; @@ -37,6 +39,10 @@ import org.junit.Test; import org.junit.runner.RunWith; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -105,25 +111,34 @@ public void testMatrixMultChainOptSparseRewrites() { private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { boolean oldFlag1 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; boolean oldFlag2 = OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES; + DMLConfig oldDMLConfig = ConfigurationManager.getDMLConfig(); try { TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); + try { + DMLConfig dmlConfig = new DMLConfig(getCurConfigFile().getPath()); + dmlConfig.setTextValue(DMLConfig.SPARSITY_REWRITES, String.valueOf(rewrites)); + overwriteCurrentConfig(dmlConfig); + } + catch(FileNotFoundException fnfe) { + Assert.fail("Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + } + catch(IOException ioe) { + Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); + } + String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + ".dml"; - if(rewrites) - programArgs = new String[] {"-explain", "hops", "-stats", - "-sparsityRewrite", "-sparsityEstimator", "Avg", - "-args", input("X"), input("Y"), output("R")}; - else - programArgs = new String[] {"-explain", "hops", "-stats", - "-args", input("X"), input("Y"), output("R")}; + programArgs = new String[] {"-explain", "hops", "-stats", + "-args", input("X"), input("Y"), output("R")}; fullRScriptName = HOME + TEST_NAME + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = rewrites; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = rewrites; + double[][] X = getRandomMatrix(rows, cols, -1, 1, sparsities[0], 7); double[][] Y = getRandomMatrix(cols, 1, -1, 1, sparsities[1], 3); long X_nnz = Stream.of(X).mapToLong(row -> DoubleStream.of(row).filter(val -> val != 0).count()).sum(); @@ -172,7 +187,17 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { finally { OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag1; OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES = oldFlag2; + try { + overwriteCurrentConfig(oldDMLConfig); + } + catch(IOException ioe) { + Assert.fail("Unable to restore the previous DML configuration. " + ioe.getMessage()); + } Recompiler.reinitRecompiler(); } } + + private void overwriteCurrentConfig(DMLConfig config) throws IOException { + Files.write(getCurConfigFile().toPath(), config.serializeDMLConfig().getBytes(StandardCharsets.UTF_8)); + } } From 5aa7f0e409582ea1bd880b11e7eeffbc280d4ed6 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 6 Aug 2026 17:03:54 +0200 Subject: [PATCH 4/5] chore(**): minor formatting --- src/main/java/org/apache/sysds/hops/OptimizerUtils.java | 3 +-- .../rewrite/RewriteMatrixMultChainOptimizationSparse.java | 4 ++-- .../rewrite/RewriteMatrixMultChainOptSparseTest.java | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java index a364f394de7..4c5edbdcc28 100644 --- a/src/main/java/org/apache/sysds/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysds/hops/OptimizerUtils.java @@ -200,8 +200,7 @@ public enum MemoryManager { public static boolean ALLOW_SUM_PRODUCT_REWRITES2 = true; /** - * Enables transpose mmchain optimizations. In the future, this might be merged with - * ALLOW_SUM_PRODUCT_REWRITES. + * Enables transpose mmchain optimizations. In the future, this might be merged with ALLOW_SUM_PRODUCT_REWRITES. */ public static boolean ALLOW_TRANSPOSE_MMCHAIN_REWRITES = false; diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java index 865157c6323..5ab9d57e44c 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteMatrixMultChainOptimizationSparse.java @@ -90,8 +90,8 @@ private static int[][] mmChainDPSparse(double[] dimArray, MMNode[] sketchArray, //compute cost-optimal chains for increasing chain sizes SparsityEstimator estim = EstimatorType.valueOf(ConfigurationManager.getDMLConfig() .getTextValue(DMLConfig.SPARSITY_ESTIMATOR)).getEstimator(); - for( int l = 2; l <= size; l++ ) { // chain length - for( int i = 0; i < size - l + 1; i++ ) { + for(int l = 2; l <= size; l++) { // chain length + for(int i = 0; i < size - l + 1; i++) { int j = i + l - 1; // find cost of (i,j) dpMatrix[i][j] = Double.MAX_VALUE; diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java index 2c7be326c95..bf9acd9e52a 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixMultChainOptSparseTest.java @@ -123,7 +123,8 @@ private void testRewriteMatrixMultChainOpSparse(boolean rewrites) { overwriteCurrentConfig(dmlConfig); } catch(FileNotFoundException fnfe) { - Assert.fail("Could not find DML config file: " + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); } catch(IOException ioe) { Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); From 6ebaec342717e2ccdd28a406528811fb3fb796df Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Fri, 7 Aug 2026 12:02:59 +0200 Subject: [PATCH 5/5] chore(test/functions/rewrite/RewriteMatrixChainDPTest.java): re-activate rewrites based on sparsity estimates for the rewrite matrix chain dynamic programming test --- .../rewrite/RewriteMatrixChainDPTest.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java index a975efe7aa0..60b491b8141 100644 --- a/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java +++ b/src/test/java/org/apache/sysds/test/functions/rewrite/RewriteMatrixChainDPTest.java @@ -21,7 +21,15 @@ import org.junit.Assert; import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.conf.DMLConfig; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.recompile.Recompiler; import org.apache.sysds.test.AutomatedTestBase; @@ -123,9 +131,10 @@ public void setUp() { private void runTestMatrixChainDP(String testName) { ExecMode platformOld = rtplatform; - boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; - boolean newMMchain1 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; - boolean newMMchain2 = OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE; + boolean oldFlag1 = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + boolean oldFlag2 = OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES; + boolean oldFlag3 = OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE; + DMLConfig oldDMLConfig = ConfigurationManager.getDMLConfig(); try { rtplatform = ExecMode.SINGLE_NODE; @@ -136,6 +145,19 @@ private void runTestMatrixChainDP(String testName) { TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); + try { + DMLConfig dmlConfig = new DMLConfig(getCurConfigFile().getPath()); + dmlConfig.setTextValue(DMLConfig.SPARSITY_REWRITES, "true"); + overwriteCurrentConfig(dmlConfig); + } + catch(FileNotFoundException fnfe) { + Assert.fail("Could not find DML config file: " + + getCurConfigFile().getPath() + " . " + fnfe.getMessage()); + } + catch(IOException ioe) { + Assert.fail("Could not overwrite the DML configuration file. " + ioe.getMessage()); + } + String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; @@ -300,11 +322,21 @@ private void runTestMatrixChainDP(String testName) { } } } finally { - OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld; - OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = newMMchain1; - OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = newMMchain2; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag1; + OptimizerUtils.ALLOW_TRANSPOSE_MMCHAIN_REWRITES = oldFlag2; + OptimizerUtils.ALLOW_NEW_MMCHAIN_REWRITE = oldFlag3; + try { + overwriteCurrentConfig(oldDMLConfig); + } + catch(IOException ioe) { + Assert.fail("Unable to restore the previous DML configuration. " + ioe.getMessage()); + } rtplatform = platformOld; Recompiler.reinitRecompiler(); } } + + private void overwriteCurrentConfig(DMLConfig config) throws IOException { + Files.write(getCurConfigFile().toPath(), config.serializeDMLConfig().getBytes(StandardCharsets.UTF_8)); + } }