feat(regex): make callback mutations transactional

dev/design/phase36-regex-parity.md

@@ -241,11 +241,13 @@ timing delta is a regression only after a serialized same-commit reproduction.
 ### Current Status: Stage 36.4 in progress
 
 The merged Joni dynamic-pattern engine establishes the Stage 36.5 execution
-seam. The current same-commit Stage 36.4 core baseline is `rxcode.t` 39/42 and
-`reg_eval_scope.t` 22/49, with no timeout or incomplete file. Callback
-exceptions now restore dynamic locals, provisional match state, and `$^R` on
-both execution backends even though the matcher cannot create an unwind token
-for a callout that throws.
+seam. The current Stage 36.4 core baseline is `rxcode.t` 42/42 and
+`reg_eval_scope.t` 22/49, with no timeout or incomplete file. Matcher-owned
+transactions now restore ordinary scalar, array, and hash mutations when the
+overall match fails, retain mutations from abandoned alternatives when another
+alternative succeeds, and commit successful matches. Callback exceptions also
+restore dynamic locals, provisional match state, and `$^R` on both execution
+backends. Regex stringification no longer exposes private callback IDs.
 
 ### Completed stages
 
@@ -260,22 +262,22 @@ for a callout that throws.
 
 ### Next steps
 
-1. Add matcher-owned mutation checkpoints so writes made by a callback on a
-   path that later backtracks are restored. This is the direct blocker for
-   `rxcode.t` assertions 26 and 34.
-2. Complete callback lexical pragma, caller-frame, and control-flow isolation
+1. Complete callback lexical pragma, caller-frame, and control-flow isolation
    exposed by `reg_eval_scope.t`, without changing its thread wrapper.
-3. Extend the callback semantic matrix with interruption, timeout, and nested
+2. Extend the callback semantic matrix with interruption, timeout, nested
    exception paths; require identical JVM/interpreter cleanup.
+3. Define and implement the mutation policy for tied, magical, shared, and
+   readonly values; ordinary values are now transactionally covered.
 4. Complete the merged dynamic-pattern validation gates, then mark Stage 36.5
    complete and proceed to the remaining declarative parity slices.
 
 ### Open blockers
 
-- Several callback-localization and dynamic-pattern capture rules still require
-  standard-Perl differential evidence.
-- Ordinary scalar and aggregate mutations performed by a callback are not yet
-  transactionally restored when the matcher abandons that path.
+- Callback lexical pragmata, caller frames, and non-local control-flow
+  boundaries still differ from Perl in `reg_eval_scope.t`.
+- Tied, magical, shared, and readonly callback mutation rollback remains
+  intentionally outside the ordinary-value transaction until its exact Perl
+  behavior is established with differential tests.
 - Dynamic `(??{ EXPR })` execution is integrated, but its full Stage 36.5 CPAN
   and unchanged-core exit matrix is not yet recorded.
 - Partial direct core tests still contain diagnostic, parser, Unicode, and

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -494,14 +494,25 @@ private record Token(int localLevel, RegexState regexState, RuntimeScalar previo
         private final int[] byteToChar;
         private final List<RuntimeRegexCallback> callbacks;
         private final RegexFlags outerFlags;
+        private final RegexCallbackMutationSnapshot mutations;
         private RuntimeScalar completedResult;
 
         PerlCalloutHandler(String input, int[] byteToChar, List<RuntimeRegexCallback> callbacks,
                            RegexFlags outerFlags) {
+            this(input, byteToChar, callbacks, outerFlags,
+                    RegexCallbackMutationSnapshot.capture());
+        }
+
+        private PerlCalloutHandler(String input, int[] byteToChar,
+                                   List<RuntimeRegexCallback> callbacks,
+                                   RegexFlags outerFlags,
+                                   RegexCallbackMutationSnapshot mutations) {
             this.input = input;
             this.byteToChar = byteToChar;
             this.callbacks = callbacks;
             this.outerFlags = outerFlags;
+            this.mutations = mutations;
+            for (RuntimeRegexCallback callback : callbacks) mutations.include(callback.code);
         }
 
         @Override
@@ -544,7 +555,8 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) {
                     : new PerlCalloutHandler(input, byteToChar, nestedCallbacks,
                             value.value instanceof RuntimeRegex runtimeRegex
                                     && runtimeRegex.getRegexFlags() != null
-                                    ? runtimeRegex.getRegexFlags() : outerFlags);
+                                    ? runtimeRegex.getRegexFlags() : outerFlags,
+                            mutations);
             return new DynamicPatternResult(nestedPattern.engineRegex(), nestedHandler,
                     evaluation.token());
         }
@@ -557,19 +569,22 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) {
             RuntimeScalar rVariable = GlobalVariable.getGlobalVariable(
                     GlobalContext.encodeSpecialVar("R"));
             RuntimeScalar previousR = rVariable.clone();
+            mutations.include(callback.code);
             publishProvisional(match);
 
             try {
                 RuntimeScalar result = RuntimeCode.apply(new RuntimeScalar(callback.code),
                         new RuntimeArray(), RuntimeContextType.SCALAR).scalar();
                 boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK;
                 if (block) rVariable.set(result);
-                Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block);
+                Token token = new Token(localLevel, savedRegex, previousR,
+                        result.clone(), block);
                 return new Evaluation(result, token);
             } catch (RuntimeException | Error failure) {
                 // The matcher cannot register an unwind token when the callout
                 // itself throws. Restore the provisional match and dynamic
                 // scope here before the exception crosses an eval boundary.
+                mutations.restore();
                 restoreCallbackScope(localLevel, savedRegex, previousR);
                 throw failure;
             }
@@ -586,6 +601,7 @@ public void complete(Object value) {
         }
 
         void finish(boolean matched) {
+            if (!matched) mutations.restore();
             if (matched && completedResult != null) {
                 GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R"))
                         .set(completedResult);

src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java

@@ -3066,6 +3066,13 @@ private static ResolvedRegex resolveRegexWithOrigin(RuntimeScalar quotedRegex) {
     @Override
     public String toString() {
         // Construct the Perl-like regex string with flags
+        String displayPattern = executableCallbacks.isEmpty()
+                ? patternString
+                : RuntimeRegexTemplate.displayPattern(patternString);
+        return "(?^" + regexFlags.toFlagString() + ":" + displayPattern + ")";
+    }
+
+    String toExecutableString() {
         return "(?^" + regexFlags.toFlagString() + ":" + patternString + ")";
     }
 

src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java

@@ -53,7 +53,7 @@ public static RuntimeScalar build(RuntimeList parts) {
                 }
             } else if (scalar.value instanceof RuntimeRegex regex
                     && !regex.executableCallbacks.isEmpty()) {
-                appendEmbeddedRegex(pattern, callbacks, regex.toString(), regex.executableCallbacks);
+                appendEmbeddedRegex(pattern, callbacks, regex.toExecutableString(), regex.executableCallbacks);
             } else if (scalar.value instanceof RuntimeRegexTemplate template) {
                 appendEmbeddedRegex(pattern, callbacks, template.pattern, template.callbacks);
             } else {
@@ -96,6 +96,20 @@ List<RuntimeRegexCallback> callbacks() {
         return callbacks;
     }
 
+    static String displayPattern(String executablePattern) {
+        if (executablePattern == null || executablePattern.isEmpty()) {
+            return executablePattern;
+        }
+        Matcher matcher = CALLOUT_ID.matcher(executablePattern);
+        StringBuilder display = new StringBuilder();
+        while (matcher.find()) {
+            String replacement = "DYNAMIC".equals(matcher.group(1)) ? "(??{})" : "(?{})";
+            matcher.appendReplacement(display, Matcher.quoteReplacement(replacement));
+        }
+        matcher.appendTail(display);
+        return display.toString();
+    }
+
     @Override
     public String toString() {
         return pattern;

src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java

@@ -0,0 +1,94 @@
+package org.perlonjava.runtime.runtimetypes;
+
+import java.util.ArrayDeque;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+/** Matcher-owned save stack for Perl regex callback mutations. */
+public final class RegexCallbackMutationSnapshot {
+    private final IdentityHashMap<RuntimeScalar, Object> scalars = new IdentityHashMap<>();
+    private final IdentityHashMap<RuntimeArray, Object> arrays = new IdentityHashMap<>();
+    private final IdentityHashMap<RuntimeHash, Object> hashes = new IdentityHashMap<>();
+
+    private final IdentityHashMap<RuntimeBase, Boolean> seen = new IdentityHashMap<>();
+
+    private RegexCallbackMutationSnapshot() {
+        ArrayDeque<RuntimeBase> work = new ArrayDeque<>();
+        for (Map.Entry<String, RuntimeScalar> entry : GlobalVariable.globalVariables.entrySet()) {
+            if (isOrdinaryPackageScalar(entry.getKey())) work.add(entry.getValue());
+        }
+        addAll(work, GlobalVariable.globalArrays.values());
+        addAll(work, GlobalVariable.globalHashes.values());
+        capture(work);
+    }
+
+    public void include(RuntimeCode callback) {
+        ArrayDeque<RuntimeBase> work = new ArrayDeque<>();
+        if (callback.closedOverVariables != null) addAll(work, callback.closedOverVariables.values());
+        addAll(work, callback.capturedScalars);
+        addAll(work, callback.capturedAggregates);
+        capture(work);
+    }
+
+    private void capture(ArrayDeque<RuntimeBase> work) {
+        while (!work.isEmpty()) {
+            RuntimeBase value = work.removeLast();
+            if (value == null || seen.put(value, Boolean.TRUE) != null) continue;
+            if (value instanceof RuntimeScalar scalar) {
+                Object state = scalar.snapshotRegexMutationState();
+                if (state != null) scalars.put(scalar, state);
+                if (scalar.value instanceof RuntimeArray array) work.add(array);
+                else if (scalar.value instanceof RuntimeHash hash) work.add(hash);
+                else if (scalar.value instanceof RuntimeScalar nested) work.add(nested);
+            } else if (value instanceof RuntimeArray array) {
+                Object state = array.snapshotRegexMutationState();
+                if (state == null) continue;
+                arrays.put(array, state);
+                addAll(work, array.elements);
+            } else if (value instanceof RuntimeHash hash) {
+                Object state = hash.snapshotRegexMutationState();
+                if (state == null) continue;
+                hashes.put(hash, state);
+                addAll(work, hash.elements.values());
+            }
+        }
+    }
+
+    public static RegexCallbackMutationSnapshot capture() {
+        return new RegexCallbackMutationSnapshot();
+    }
+
+    public void restore() {
+        for (Map.Entry<RuntimeScalar, Object> entry : scalars.entrySet()) {
+            entry.getKey().restoreRegexMutationState(entry.getValue());
+        }
+        for (Map.Entry<RuntimeArray, Object> entry : arrays.entrySet()) {
+            entry.getKey().restoreRegexMutationState(entry.getValue());
+        }
+        for (Map.Entry<RuntimeHash, Object> entry : hashes.entrySet()) {
+            entry.getKey().restoreRegexMutationState(entry.getValue());
+        }
+        MortalList.flush();
+    }
+
+    private static void addAll(ArrayDeque<RuntimeBase> work,
+                               Iterable<? extends RuntimeBase> values) {
+        if (values == null) return;
+        for (RuntimeBase value : values) if (value != null) work.add(value);
+    }
+
+    private static void addAll(ArrayDeque<RuntimeBase> work, RuntimeBase[] values) {
+        if (values == null) return;
+        for (RuntimeBase value : values) if (value != null) work.add(value);
+    }
+
+    private static boolean isOrdinaryPackageScalar(String name) {
+        int separator = name.lastIndexOf("::");
+        String symbol = separator < 0 ? name : name.substring(separator + 2);
+        if (symbol.isEmpty() || !Character.isJavaIdentifierStart(symbol.charAt(0))) return false;
+        for (int i = 1; i < symbol.length(); i++) {
+            if (!Character.isJavaIdentifierPart(symbol.charAt(i))) return false;
+        }
+        return true;
+    }
+}

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeArray.java

@@ -87,6 +87,52 @@ void resetElementListAfterAutovivification() {
         elements = newElementList();
     }
 
+    Object snapshotRegexMutationState() {
+        if (threadShared || type == TIED_ARRAY || type == READONLY_ARRAY) return null;
+        Set<RuntimeScalar> ownedAliases = ownedAliasElements == null ? null
+                : Collections.newSetFromMap(new IdentityHashMap<>());
+        if (ownedAliases != null) ownedAliases.addAll(ownedAliasElements);
+        return new RegexMutationState(new ArrayList<>(elements), type, strictAutovivify,
+                scalarContextSize, elementsOwned, elementsAliased, ownedAliases, blessId);
+    }
+
+    void restoreRegexMutationState(Object token) {
+        if (!(token instanceof RegexMutationState state)) return;
+        deferElementsAddedSince(state.elements);
+        elements = newElementList(state.elements);
+        type = state.type;
+        strictAutovivify = state.strictAutovivify;
+        scalarContextSize = state.scalarContextSize;
+        elementsOwned = state.elementsOwned;
+        elementsAliased = state.elementsAliased;
+        ownedAliasElements = state.ownedAliasElements == null ? null
+                : Collections.newSetFromMap(new IdentityHashMap<>());
+        if (ownedAliasElements != null) ownedAliasElements.addAll(state.ownedAliasElements);
+        blessId = state.blessId;
+        markPackageRootedValues(0);
+    }
+
+    private void deferElementsAddedSince(List<RuntimeScalar> saved) {
+        IdentityHashMap<RuntimeScalar, Integer> retained = new IdentityHashMap<>();
+        for (RuntimeScalar scalar : saved) {
+            if (scalar != null) retained.merge(scalar, 1, Integer::sum);
+        }
+        List<RuntimeScalar> added = new ArrayList<>();
+        for (RuntimeScalar scalar : elements) {
+            if (scalar == null) continue;
+            Integer count = retained.get(scalar);
+            if (count == null || count == 0) added.add(scalar);
+            else if (count == 1) retained.remove(scalar);
+            else retained.put(scalar, count - 1);
+        }
+        MortalList.deferDestroyForContainerClear(added);
+    }
+
+    private record RegexMutationState(List<RuntimeScalar> elements, int type,
+                                      boolean strictAutovivify, Integer scalarContextSize,
+                                      boolean elementsOwned, boolean elementsAliased,
+                                      Set<RuntimeScalar> ownedAliasElements, int blessId) {}
+
     private static final class RuntimeArrayElementList extends ArrayList<RuntimeScalar> {
         private final RuntimeArray owner;
 

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java

@@ -83,6 +83,46 @@ void resetElementMapAfterAutovivification() {
         elements = newElementMap();
     }
 
+    Object snapshotRegexMutationState() {
+        if (threadShared || type == TIED_HASH || type == READONLY_HASH) return null;
+        return new RegexMutationState(new LinkedHashMap<>(elements),
+                byteKeys == null ? null : new HashSet<>(byteKeys), type, blessId,
+                taintEnvironmentAliasDescription);
+    }
+
+    void restoreRegexMutationState(Object token) {
+        if (!(token instanceof RegexMutationState state)) return;
+        deferValuesAddedSince(state.elements.values());
+        elements = newElementMap(state.elements);
+        byteKeys = state.byteKeys == null ? null : new HashSet<>(state.byteKeys);
+        type = state.type;
+        blessId = state.blessId;
+        taintEnvironmentAliasDescription = state.taintEnvironmentAliasDescription;
+        if (isPackageRootedHash()) {
+            for (RuntimeScalar value : elements.values()) markPackageRootedValue(value);
+        }
+    }
+
+    private void deferValuesAddedSince(Collection<RuntimeScalar> saved) {
+        IdentityHashMap<RuntimeScalar, Integer> retained = new IdentityHashMap<>();
+        for (RuntimeScalar scalar : saved) {
+            if (scalar != null) retained.merge(scalar, 1, Integer::sum);
+        }
+        List<RuntimeScalar> added = new ArrayList<>();
+        for (RuntimeScalar scalar : elements.values()) {
+            if (scalar == null) continue;
+            Integer count = retained.get(scalar);
+            if (count == null || count == 0) added.add(scalar);
+            else if (count == 1) retained.remove(scalar);
+            else retained.put(scalar, count - 1);
+        }
+        MortalList.deferDestroyForContainerClear(added);
+    }
+
+    private record RegexMutationState(Map<String, RuntimeScalar> elements,
+                                      Set<String> byteKeys, int type, int blessId,
+                                      String taintEnvironmentAliasDescription) {}
+
     private static final class RuntimeHashElementMap extends StableHashMap<String, RuntimeScalar> {
         private final RuntimeHash owner;
 

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java

@@ -707,6 +707,34 @@ public RuntimeScalar clone() {
         return new RuntimeScalar(this);
     }
 
+    /** Capture a plain scalar payload for regex callback backtracking. */
+    Object snapshotRegexMutationState() {
+        if (type < INTEGER || type > BOOLEAN) return null;
+        return new RegexMutationState(type, value, utf8UncheckedOctets, tainted,
+                numericLiteralText, numericContextSeen, firstClassRegexScalar,
+                formatPictureTainted);
+    }
+
+    void restoreRegexMutationState(Object token) {
+        if (!(token instanceof RegexMutationState state)) return;
+        type = state.type;
+        value = state.value;
+        utf8UncheckedOctets = state.utf8UncheckedOctets;
+        tainted = state.tainted;
+        numericLiteralText = state.numericLiteralText;
+        numericContextSeen = state.numericContextSeen;
+        firstClassRegexScalar = state.firstClassRegexScalar;
+        formatPictureTainted = state.formatPictureTainted;
+        RuntimePosLvalue.invalidatePos(this);
+        refreshSubstrLvalues();
+    }
+
+    private record RegexMutationState(int type, Object value,
+                                      boolean utf8UncheckedOctets, boolean tainted,
+                                      String numericLiteralText, boolean numericContextSeen,
+                                      boolean firstClassRegexScalar,
+                                      boolean formatPictureTainted) {}
+
     public int countElements() {
         return 1;
     }