From c79f20f519290fd6ee81107e334bb8bd4fde3c6a Mon Sep 17 00:00:00 2001 From: "Flavio S. Glock" Date: Sun, 16 Aug 2026 11:28:04 +0200 Subject: [PATCH] feat(regex): make callback mutations transactional Snapshot ordinary callback-visible scalar, array, and hash state for each matcher attempt. Restore it when the overall match fails or a callback throws, while preserving Perl's committed mutations when another alternative succeeds. Hide internal callback IDs from regex stringification without breaking callback remapping for embedded patterns. Add standard-Perl-valid regression coverage and update the Phase 36 delivery plan with the verified 42/42 rxcode baseline. Generated with [Codex](https://openai.com/codex) Co-Authored-By: Codex --- dev/design/phase36-regex-parity.md | 30 +++--- .../runtime/regex/JoniRegexPattern.java | 20 +++- .../runtime/regex/RuntimeRegex.java | 7 ++ .../runtime/regex/RuntimeRegexTemplate.java | 16 +++- .../RegexCallbackMutationSnapshot.java | 94 +++++++++++++++++++ .../runtime/runtimetypes/RuntimeArray.java | 46 +++++++++ .../runtime/runtimetypes/RuntimeHash.java | 40 ++++++++ .../runtime/runtimetypes/RuntimeScalar.java | 28 ++++++ .../unit/regex/callback_mutation_rollback.t | 32 +++++++ .../unit/regex/callback_stringification.t | 16 ++++ 10 files changed, 312 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java create mode 100644 src/test/resources/unit/regex/callback_mutation_rollback.t create mode 100644 src/test/resources/unit/regex/callback_stringification.t diff --git a/dev/design/phase36-regex-parity.md b/dev/design/phase36-regex-parity.md index a211254dde..6da84150a2 100644 --- a/dev/design/phase36-regex-parity.md +++ b/dev/design/phase36-regex-parity.md @@ -241,11 +241,13 @@ timing delta is a regression only after a serialized same-commit reproduction. ### Current Status: Stage 36.4 in progress The merged Joni dynamic-pattern engine establishes the Stage 36.5 execution -seam. The current same-commit Stage 36.4 core baseline is `rxcode.t` 39/42 and -`reg_eval_scope.t` 22/49, with no timeout or incomplete file. Callback -exceptions now restore dynamic locals, provisional match state, and `$^R` on -both execution backends even though the matcher cannot create an unwind token -for a callout that throws. +seam. The current Stage 36.4 core baseline is `rxcode.t` 42/42 and +`reg_eval_scope.t` 22/49, with no timeout or incomplete file. Matcher-owned +transactions now restore ordinary scalar, array, and hash mutations when the +overall match fails, retain mutations from abandoned alternatives when another +alternative succeeds, and commit successful matches. Callback exceptions also +restore dynamic locals, provisional match state, and `$^R` on both execution +backends. Regex stringification no longer exposes private callback IDs. ### Completed stages @@ -260,22 +262,22 @@ for a callout that throws. ### Next steps -1. Add matcher-owned mutation checkpoints so writes made by a callback on a - path that later backtracks are restored. This is the direct blocker for - `rxcode.t` assertions 26 and 34. -2. Complete callback lexical pragma, caller-frame, and control-flow isolation +1. Complete callback lexical pragma, caller-frame, and control-flow isolation exposed by `reg_eval_scope.t`, without changing its thread wrapper. -3. Extend the callback semantic matrix with interruption, timeout, and nested +2. Extend the callback semantic matrix with interruption, timeout, nested exception paths; require identical JVM/interpreter cleanup. +3. Define and implement the mutation policy for tied, magical, shared, and + readonly values; ordinary values are now transactionally covered. 4. Complete the merged dynamic-pattern validation gates, then mark Stage 36.5 complete and proceed to the remaining declarative parity slices. ### Open blockers -- Several callback-localization and dynamic-pattern capture rules still require - standard-Perl differential evidence. -- Ordinary scalar and aggregate mutations performed by a callback are not yet - transactionally restored when the matcher abandons that path. +- Callback lexical pragmata, caller frames, and non-local control-flow + boundaries still differ from Perl in `reg_eval_scope.t`. +- Tied, magical, shared, and readonly callback mutation rollback remains + intentionally outside the ordinary-value transaction until its exact Perl + behavior is established with differential tests. - Dynamic `(??{ EXPR })` execution is integrated, but its full Stage 36.5 CPAN and unchanged-core exit matrix is not yet recorded. - Partial direct core tests still contain diagnostic, parser, Unicode, and diff --git a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java index eeac02ce7c..56eac5e795 100644 --- a/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java +++ b/src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java @@ -494,14 +494,25 @@ private record Token(int localLevel, RegexState regexState, RuntimeScalar previo private final int[] byteToChar; private final List callbacks; private final RegexFlags outerFlags; + private final RegexCallbackMutationSnapshot mutations; private RuntimeScalar completedResult; PerlCalloutHandler(String input, int[] byteToChar, List callbacks, RegexFlags outerFlags) { + this(input, byteToChar, callbacks, outerFlags, + RegexCallbackMutationSnapshot.capture()); + } + + private PerlCalloutHandler(String input, int[] byteToChar, + List callbacks, + RegexFlags outerFlags, + RegexCallbackMutationSnapshot mutations) { this.input = input; this.byteToChar = byteToChar; this.callbacks = callbacks; this.outerFlags = outerFlags; + this.mutations = mutations; + for (RuntimeRegexCallback callback : callbacks) mutations.include(callback.code); } @Override @@ -544,7 +555,8 @@ public DynamicPatternResult executeDynamic(int id, MatchView match) { : new PerlCalloutHandler(input, byteToChar, nestedCallbacks, value.value instanceof RuntimeRegex runtimeRegex && runtimeRegex.getRegexFlags() != null - ? runtimeRegex.getRegexFlags() : outerFlags); + ? runtimeRegex.getRegexFlags() : outerFlags, + mutations); return new DynamicPatternResult(nestedPattern.engineRegex(), nestedHandler, evaluation.token()); } @@ -557,6 +569,7 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { RuntimeScalar rVariable = GlobalVariable.getGlobalVariable( GlobalContext.encodeSpecialVar("R")); RuntimeScalar previousR = rVariable.clone(); + mutations.include(callback.code); publishProvisional(match); try { @@ -564,12 +577,14 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) { new RuntimeArray(), RuntimeContextType.SCALAR).scalar(); boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK; if (block) rVariable.set(result); - Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block); + Token token = new Token(localLevel, savedRegex, previousR, + result.clone(), block); return new Evaluation(result, token); } catch (RuntimeException | Error failure) { // The matcher cannot register an unwind token when the callout // itself throws. Restore the provisional match and dynamic // scope here before the exception crosses an eval boundary. + mutations.restore(); restoreCallbackScope(localLevel, savedRegex, previousR); throw failure; } @@ -586,6 +601,7 @@ public void complete(Object value) { } void finish(boolean matched) { + if (!matched) mutations.restore(); if (matched && completedResult != null) { GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R")) .set(completedResult); diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java index de0433e7c3..126def0da0 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java @@ -3066,6 +3066,13 @@ private static ResolvedRegex resolveRegexWithOrigin(RuntimeScalar quotedRegex) { @Override public String toString() { // Construct the Perl-like regex string with flags + String displayPattern = executableCallbacks.isEmpty() + ? patternString + : RuntimeRegexTemplate.displayPattern(patternString); + return "(?^" + regexFlags.toFlagString() + ":" + displayPattern + ")"; + } + + String toExecutableString() { return "(?^" + regexFlags.toFlagString() + ":" + patternString + ")"; } diff --git a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java index b4721ead88..721e16e8d8 100644 --- a/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java +++ b/src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java @@ -53,7 +53,7 @@ public static RuntimeScalar build(RuntimeList parts) { } } else if (scalar.value instanceof RuntimeRegex regex && !regex.executableCallbacks.isEmpty()) { - appendEmbeddedRegex(pattern, callbacks, regex.toString(), regex.executableCallbacks); + appendEmbeddedRegex(pattern, callbacks, regex.toExecutableString(), regex.executableCallbacks); } else if (scalar.value instanceof RuntimeRegexTemplate template) { appendEmbeddedRegex(pattern, callbacks, template.pattern, template.callbacks); } else { @@ -96,6 +96,20 @@ List callbacks() { return callbacks; } + static String displayPattern(String executablePattern) { + if (executablePattern == null || executablePattern.isEmpty()) { + return executablePattern; + } + Matcher matcher = CALLOUT_ID.matcher(executablePattern); + StringBuilder display = new StringBuilder(); + while (matcher.find()) { + String replacement = "DYNAMIC".equals(matcher.group(1)) ? "(??{})" : "(?{})"; + matcher.appendReplacement(display, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(display); + return display.toString(); + } + @Override public String toString() { return pattern; diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java new file mode 100644 index 0000000000..5b192d1067 --- /dev/null +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RegexCallbackMutationSnapshot.java @@ -0,0 +1,94 @@ +package org.perlonjava.runtime.runtimetypes; + +import java.util.ArrayDeque; +import java.util.IdentityHashMap; +import java.util.Map; + +/** Matcher-owned save stack for Perl regex callback mutations. */ +public final class RegexCallbackMutationSnapshot { + private final IdentityHashMap scalars = new IdentityHashMap<>(); + private final IdentityHashMap arrays = new IdentityHashMap<>(); + private final IdentityHashMap hashes = new IdentityHashMap<>(); + + private final IdentityHashMap seen = new IdentityHashMap<>(); + + private RegexCallbackMutationSnapshot() { + ArrayDeque work = new ArrayDeque<>(); + for (Map.Entry entry : GlobalVariable.globalVariables.entrySet()) { + if (isOrdinaryPackageScalar(entry.getKey())) work.add(entry.getValue()); + } + addAll(work, GlobalVariable.globalArrays.values()); + addAll(work, GlobalVariable.globalHashes.values()); + capture(work); + } + + public void include(RuntimeCode callback) { + ArrayDeque work = new ArrayDeque<>(); + if (callback.closedOverVariables != null) addAll(work, callback.closedOverVariables.values()); + addAll(work, callback.capturedScalars); + addAll(work, callback.capturedAggregates); + capture(work); + } + + private void capture(ArrayDeque work) { + while (!work.isEmpty()) { + RuntimeBase value = work.removeLast(); + if (value == null || seen.put(value, Boolean.TRUE) != null) continue; + if (value instanceof RuntimeScalar scalar) { + Object state = scalar.snapshotRegexMutationState(); + if (state != null) scalars.put(scalar, state); + if (scalar.value instanceof RuntimeArray array) work.add(array); + else if (scalar.value instanceof RuntimeHash hash) work.add(hash); + else if (scalar.value instanceof RuntimeScalar nested) work.add(nested); + } else if (value instanceof RuntimeArray array) { + Object state = array.snapshotRegexMutationState(); + if (state == null) continue; + arrays.put(array, state); + addAll(work, array.elements); + } else if (value instanceof RuntimeHash hash) { + Object state = hash.snapshotRegexMutationState(); + if (state == null) continue; + hashes.put(hash, state); + addAll(work, hash.elements.values()); + } + } + } + + public static RegexCallbackMutationSnapshot capture() { + return new RegexCallbackMutationSnapshot(); + } + + public void restore() { + for (Map.Entry entry : scalars.entrySet()) { + entry.getKey().restoreRegexMutationState(entry.getValue()); + } + for (Map.Entry entry : arrays.entrySet()) { + entry.getKey().restoreRegexMutationState(entry.getValue()); + } + for (Map.Entry entry : hashes.entrySet()) { + entry.getKey().restoreRegexMutationState(entry.getValue()); + } + MortalList.flush(); + } + + private static void addAll(ArrayDeque work, + Iterable values) { + if (values == null) return; + for (RuntimeBase value : values) if (value != null) work.add(value); + } + + private static void addAll(ArrayDeque work, RuntimeBase[] values) { + if (values == null) return; + for (RuntimeBase value : values) if (value != null) work.add(value); + } + + private static boolean isOrdinaryPackageScalar(String name) { + int separator = name.lastIndexOf("::"); + String symbol = separator < 0 ? name : name.substring(separator + 2); + if (symbol.isEmpty() || !Character.isJavaIdentifierStart(symbol.charAt(0))) return false; + for (int i = 1; i < symbol.length(); i++) { + if (!Character.isJavaIdentifierPart(symbol.charAt(i))) return false; + } + return true; + } +} diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeArray.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeArray.java index 0597c605a5..9c8b7a328c 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeArray.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeArray.java @@ -87,6 +87,52 @@ void resetElementListAfterAutovivification() { elements = newElementList(); } + Object snapshotRegexMutationState() { + if (threadShared || type == TIED_ARRAY || type == READONLY_ARRAY) return null; + Set ownedAliases = ownedAliasElements == null ? null + : Collections.newSetFromMap(new IdentityHashMap<>()); + if (ownedAliases != null) ownedAliases.addAll(ownedAliasElements); + return new RegexMutationState(new ArrayList<>(elements), type, strictAutovivify, + scalarContextSize, elementsOwned, elementsAliased, ownedAliases, blessId); + } + + void restoreRegexMutationState(Object token) { + if (!(token instanceof RegexMutationState state)) return; + deferElementsAddedSince(state.elements); + elements = newElementList(state.elements); + type = state.type; + strictAutovivify = state.strictAutovivify; + scalarContextSize = state.scalarContextSize; + elementsOwned = state.elementsOwned; + elementsAliased = state.elementsAliased; + ownedAliasElements = state.ownedAliasElements == null ? null + : Collections.newSetFromMap(new IdentityHashMap<>()); + if (ownedAliasElements != null) ownedAliasElements.addAll(state.ownedAliasElements); + blessId = state.blessId; + markPackageRootedValues(0); + } + + private void deferElementsAddedSince(List saved) { + IdentityHashMap retained = new IdentityHashMap<>(); + for (RuntimeScalar scalar : saved) { + if (scalar != null) retained.merge(scalar, 1, Integer::sum); + } + List added = new ArrayList<>(); + for (RuntimeScalar scalar : elements) { + if (scalar == null) continue; + Integer count = retained.get(scalar); + if (count == null || count == 0) added.add(scalar); + else if (count == 1) retained.remove(scalar); + else retained.put(scalar, count - 1); + } + MortalList.deferDestroyForContainerClear(added); + } + + private record RegexMutationState(List elements, int type, + boolean strictAutovivify, Integer scalarContextSize, + boolean elementsOwned, boolean elementsAliased, + Set ownedAliasElements, int blessId) {} + private static final class RuntimeArrayElementList extends ArrayList { private final RuntimeArray owner; diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java index b1b088503d..78f08b5b7f 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeHash.java @@ -83,6 +83,46 @@ void resetElementMapAfterAutovivification() { elements = newElementMap(); } + Object snapshotRegexMutationState() { + if (threadShared || type == TIED_HASH || type == READONLY_HASH) return null; + return new RegexMutationState(new LinkedHashMap<>(elements), + byteKeys == null ? null : new HashSet<>(byteKeys), type, blessId, + taintEnvironmentAliasDescription); + } + + void restoreRegexMutationState(Object token) { + if (!(token instanceof RegexMutationState state)) return; + deferValuesAddedSince(state.elements.values()); + elements = newElementMap(state.elements); + byteKeys = state.byteKeys == null ? null : new HashSet<>(state.byteKeys); + type = state.type; + blessId = state.blessId; + taintEnvironmentAliasDescription = state.taintEnvironmentAliasDescription; + if (isPackageRootedHash()) { + for (RuntimeScalar value : elements.values()) markPackageRootedValue(value); + } + } + + private void deferValuesAddedSince(Collection saved) { + IdentityHashMap retained = new IdentityHashMap<>(); + for (RuntimeScalar scalar : saved) { + if (scalar != null) retained.merge(scalar, 1, Integer::sum); + } + List added = new ArrayList<>(); + for (RuntimeScalar scalar : elements.values()) { + if (scalar == null) continue; + Integer count = retained.get(scalar); + if (count == null || count == 0) added.add(scalar); + else if (count == 1) retained.remove(scalar); + else retained.put(scalar, count - 1); + } + MortalList.deferDestroyForContainerClear(added); + } + + private record RegexMutationState(Map elements, + Set byteKeys, int type, int blessId, + String taintEnvironmentAliasDescription) {} + private static final class RuntimeHashElementMap extends StableHashMap { private final RuntimeHash owner; diff --git a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java index e4ddd0ebb1..d21c927b36 100644 --- a/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java +++ b/src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeScalar.java @@ -707,6 +707,34 @@ public RuntimeScalar clone() { return new RuntimeScalar(this); } + /** Capture a plain scalar payload for regex callback backtracking. */ + Object snapshotRegexMutationState() { + if (type < INTEGER || type > BOOLEAN) return null; + return new RegexMutationState(type, value, utf8UncheckedOctets, tainted, + numericLiteralText, numericContextSeen, firstClassRegexScalar, + formatPictureTainted); + } + + void restoreRegexMutationState(Object token) { + if (!(token instanceof RegexMutationState state)) return; + type = state.type; + value = state.value; + utf8UncheckedOctets = state.utf8UncheckedOctets; + tainted = state.tainted; + numericLiteralText = state.numericLiteralText; + numericContextSeen = state.numericContextSeen; + firstClassRegexScalar = state.firstClassRegexScalar; + formatPictureTainted = state.formatPictureTainted; + RuntimePosLvalue.invalidatePos(this); + refreshSubstrLvalues(); + } + + private record RegexMutationState(int type, Object value, + boolean utf8UncheckedOctets, boolean tainted, + String numericLiteralText, boolean numericContextSeen, + boolean firstClassRegexScalar, + boolean formatPictureTainted) {} + public int countElements() { return 1; } diff --git a/src/test/resources/unit/regex/callback_mutation_rollback.t b/src/test/resources/unit/regex/callback_mutation_rollback.t new file mode 100644 index 0000000000..2a25bf5d50 --- /dev/null +++ b/src/test/resources/unit/regex/callback_mutation_rollback.t @@ -0,0 +1,32 @@ +use strict; +use warnings; + +print "1..9\n"; + +my ($scalar, @array, %hash) = (0, (), ()); +my $failed = 'ac' =~ /^a(?{ + $scalar = 1; + push @array, 2; + $hash{x} = 3; +})b$/; +print !$failed ? "ok 1\n" : "not ok 1\n"; +print $scalar == 0 ? "ok 2\n" : "not ok 2\n"; +print @array == 0 ? "ok 3\n" : "not ok 3\n"; +print !exists $hash{x} ? "ok 4\n" : "not ok 4\n"; + +my ($committed_scalar, @committed_array, %committed_hash) = (0, (), ()); +my $matched = 'ab' =~ /^a(?{ + $committed_scalar = 4; + push @committed_array, 5; + $committed_hash{x} = 6; +})b$/; +print $matched ? "ok 5\n" : "not ok 5\n"; +print $committed_scalar == 4 ? "ok 6\n" : "not ok 6\n"; +print join(',', @committed_array) eq '5' ? "ok 7\n" : "not ok 7\n"; +print $committed_hash{x} == 6 ? "ok 8\n" : "not ok 8\n"; + +my @alternative; +'ac' =~ /^(?:a(?{ push @alternative, 'abandoned' })b|a(?{ + push @alternative, 'chosen' +})c)$/; +print join(',', @alternative) eq 'abandoned,chosen' ? "ok 9\n" : "not ok 9\n"; diff --git a/src/test/resources/unit/regex/callback_stringification.t b/src/test/resources/unit/regex/callback_stringification.t new file mode 100644 index 0000000000..c46d5cf3ca --- /dev/null +++ b/src/test/resources/unit/regex/callback_stringification.t @@ -0,0 +1,16 @@ +use strict; +use warnings; +use re 'eval'; + +print "1..2\n"; + +my @values = (1, 2, 3); +my $plain = qr/@values(?{})/; +print "$plain" =~ /\Q1 2 3(?{})\E/ + ? "ok 1 - callback marker is hidden in regex stringification\n" + : "not ok 1 - callback marker is hidden in regex stringification\n"; + +my $dynamic = qr/(??{ 'x' })/; +print "$dynamic" !~ /DYNAMIC:\d+/ + ? "ok 2 - dynamic callback marker is hidden in regex stringification\n" + : "not ok 2 - dynamic callback marker is hidden in regex stringification\n";