fix: restore post-Joni core behavior

dev/design/phase36-regex-parity.md

@@ -240,6 +240,13 @@ timing delta is a regression only after a serialized same-commit reproduction.
 
 ### Current Status: Stage 36.4 in progress
 
+The merged Joni dynamic-pattern engine establishes the Stage 36.5 execution
+seam. The current same-commit Stage 36.4 core baseline is `rxcode.t` 39/42 and
+`reg_eval_scope.t` 22/49, with no timeout or incomplete file. Callback
+exceptions now restore dynamic locals, provisional match state, and `$^R` on
+both execution backends even though the matcher cannot create an unwind token
+for a callout that throws.
+
 ### Completed stages
 
 - [x] Stage 36.0: Refresh differential baseline
@@ -253,18 +260,24 @@ timing delta is a regression only after a serialized same-commit reproduction.
 
 ### Next steps
 
-1. Refresh direct and wrapper counts on the merged callout engine.
-2. Extend the callback semantic matrix with dynamic-local unwind, exceptions,
-   interruption, timeout, and nested matches.
-3. Complete Stage 36.4 without changing thread wrappers.
-4. Implement dynamic patterns and then the remaining declarative parity slices.
+1. Add matcher-owned mutation checkpoints so writes made by a callback on a
+   path that later backtracks are restored. This is the direct blocker for
+   `rxcode.t` assertions 26 and 34.
+2. Complete callback lexical pragma, caller-frame, and control-flow isolation
+   exposed by `reg_eval_scope.t`, without changing its thread wrapper.
+3. Extend the callback semantic matrix with interruption, timeout, and nested
+   exception paths; require identical JVM/interpreter cleanup.
+4. Complete the merged dynamic-pattern validation gates, then mark Stage 36.5
+   complete and proceed to the remaining declarative parity slices.
 
 ### Open blockers
 
 - Several callback-localization and dynamic-pattern capture rules still require
   standard-Perl differential evidence.
-- Dynamic `(??{ EXPR })`, optimistic callback execution, and several declarative
-  controls are not complete.
+- Ordinary scalar and aggregate mutations performed by a callback are not yet
+  transactionally restored when the matcher abandons that path.
+- Dynamic `(??{ EXPR })` execution is integrated, but its full Stage 36.5 CPAN
+  and unchanged-core exit matrix is not yet recorded.
 - Partial direct core tests still contain diagnostic, parser, Unicode, and
   regex-object gaps; their wrappers must not be mistaken for thread failures.
 

src/main/java/org/perlonjava/runtime/operators/FileTestOperator.java

@@ -361,6 +361,14 @@ public static RuntimeScalar fileTest(String operator, RuntimeScalar fileHandle)
                     return scalarUndef;
                 }
                 int fd = descriptor.getInt();
+                // Scalar-backed and other descriptorless handles report -1.
+                // Passing that sentinel to isatty() turns Perl's EBADF/undef
+                // result into a defined false value and clears $!.
+                if (fd < 0) {
+                    getGlobalVariable("main::!").set(9);
+                    updateLastStat(fileHandle, false, 9);
+                    return scalarUndef;
+                }
                 try {
                     boolean isTty = FFMPosix.get().isatty(fd) != 0;
                     getGlobalVariable("main::!").set(0);

src/main/java/org/perlonjava/runtime/operators/ListOperators.java

@@ -10,6 +10,19 @@
 import static org.perlonjava.runtime.runtimetypes.RuntimeScalarCache.scalarTrue;
 
 public class ListOperators {
+    /**
+     * Perl evaluates the input list before entering a map/grep-style block.
+     * Keep the scalar objects (and therefore their aliasing) but detach the
+     * iteration order from a source array that the block may mutate.
+     */
+    private static List<RuntimeScalar> snapshotElements(RuntimeList runtimeList) {
+        List<RuntimeScalar> snapshot = new ArrayList<>();
+        for (RuntimeScalar element : runtimeList) {
+            snapshot.add(element);
+        }
+        return snapshot;
+    }
+
     /**
      * Eagerly release captured variable references from an ephemeral grep/map/all/any
      * block closure. Like eval BLOCK closures, these blocks execute and are immediately
@@ -55,8 +68,8 @@ public static RuntimeList map(RuntimeList runtimeList, RuntimeScalar perlMapClos
             // This allows $_[0], $_[1], etc. to work inside map blocks
             RuntimeArray mapArgs = outerArgs != null ? outerArgs : new RuntimeArray();
 
-            // Iterate over each element in the current RuntimeArray
-            for (RuntimeScalar element : runtimeList) {
+            // Iterate over the list value captured before the block starts.
+            for (RuntimeScalar element : snapshotElements(runtimeList)) {
                 // Create $_ argument for the map subroutine
                 GlobalVariable.aliasTemporaryGlobalVariable("main::_", element);
 
@@ -244,8 +257,8 @@ public static RuntimeList grep(RuntimeList runtimeList, RuntimeScalar perlFilter
             // Use the outer @_ instead of an empty array
             RuntimeArray filterArgs = outerArgs != null ? outerArgs : new RuntimeArray();
 
-            // Iterate over each element in the current RuntimeArray
-            for (RuntimeScalar element : runtimeList) {
+            // Iterate over the list value captured before the block starts.
+            for (RuntimeScalar element : snapshotElements(runtimeList)) {
                 try {
                     // Create $_ argument for the filter subroutine
                     GlobalVariable.aliasTemporaryGlobalVariable("main::_", element);
@@ -320,8 +333,8 @@ public static RuntimeList all(RuntimeList runtimeList, RuntimeScalar perlFilterC
         try {
             RuntimeArray filterArgs = outerArgs != null ? outerArgs : new RuntimeArray();
 
-            // Iterate over each element in the current RuntimeArray
-            for (RuntimeScalar element : runtimeList) {
+            // Iterate over the list value captured before the block starts.
+            for (RuntimeScalar element : snapshotElements(runtimeList)) {
                 try {
                     // Create $_ argument for the filter subroutine
                     GlobalVariable.aliasTemporaryGlobalVariable("main::_", element);
@@ -380,8 +393,8 @@ public static RuntimeList any(RuntimeList runtimeList, RuntimeScalar perlFilterC
         try {
             RuntimeArray filterArgs = outerArgs != null ? outerArgs : new RuntimeArray();
 
-            // Iterate over each element in the current RuntimeArray
-            for (RuntimeScalar element : runtimeList) {
+            // Iterate over the list value captured before the block starts.
+            for (RuntimeScalar element : snapshotElements(runtimeList)) {
                 try {
                     // Create $_ argument for the filter subroutine
                     GlobalVariable.aliasTemporaryGlobalVariable("main::_", element);

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -558,12 +558,20 @@ private Evaluation evaluate(RuntimeRegexCallback callback, MatchView match) {
             RuntimeScalar previousR = rVariable.clone();
             publishProvisional(match);
 
-            RuntimeScalar result = RuntimeCode.apply(new RuntimeScalar(callback.code),
-                    new RuntimeArray(), RuntimeContextType.SCALAR).scalar();
-            boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK;
-            if (block) rVariable.set(result);
-            Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block);
-            return new Evaluation(result, token);
+            try {
+                RuntimeScalar result = RuntimeCode.apply(new RuntimeScalar(callback.code),
+                        new RuntimeArray(), RuntimeContextType.SCALAR).scalar();
+                boolean block = callback.kind == RuntimeRegexCallback.Kind.BLOCK;
+                if (block) rVariable.set(result);
+                Token token = new Token(localLevel, savedRegex, previousR, result.clone(), block);
+                return new Evaluation(result, token);
+            } catch (RuntimeException | Error failure) {
+                // The matcher cannot register an unwind token when the callout
+                // itself throws. Restore the provisional match and dynamic
+                // scope here before the exception crosses an eval boundary.
+                restoreCallbackScope(localLevel, savedRegex, previousR);
+                throw failure;
+            }
         }
 
         @Override
@@ -584,15 +592,23 @@ void finish(boolean matched) {
         }
 
         private void restore(Token token, boolean completed) {
-            DynamicVariableManager.popToLocalLevel(token.localLevel());
-            token.regexState().restore();
-            GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R"))
-                    .set(token.previousR());
+            restoreCallbackScope(token.localLevel(), token.regexState(), token.previousR());
             if (completed && token.block() && completedResult == null) {
                 completedResult = token.result();
             }
         }
 
+        private static void restoreCallbackScope(int localLevel, RegexState regexState,
+                                                 RuntimeScalar previousR) {
+            try {
+                DynamicVariableManager.popToLocalLevel(localLevel);
+            } finally {
+                regexState.restore();
+                GlobalVariable.getGlobalVariable(GlobalContext.encodeSpecialVar("R"))
+                        .set(previousR);
+            }
+        }
+
         private void publishProvisional(MatchView match) {
             RuntimeRegexState state = PerlRuntime.current().regexState;
             int count = match.captureCount();

src/main/java/org/perlonjava/runtime/regex/RuntimeRegexTemplate.java

@@ -3,6 +3,7 @@
 import org.perlonjava.runtime.runtimetypes.RuntimeBase;
 import org.perlonjava.runtime.runtimetypes.RuntimeList;
 import org.perlonjava.runtime.runtimetypes.RuntimeScalar;
+import org.perlonjava.runtime.runtimetypes.RuntimeScalarType;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -34,6 +35,11 @@ public static RuntimeScalar build(RuntimeList parts) {
         boolean tainted = false;
         for (RuntimeBase part : parts.elements) {
             RuntimeScalar scalar = part.scalar();
+            // Interpolation is one scalar read. Resolve tied magic once, then
+            // use that materialized value for type inspection, taint, and text.
+            if (scalar.type == RuntimeScalarType.TIED_SCALAR) {
+                scalar = scalar.tiedFetch();
+            }
             tainted |= scalar.isTainted();
             if (scalar.value instanceof RuntimeRegexCallback callback) {
                 int id = callbacks.size();

src/test/resources/unit/map_source_mutation_snapshot.t

@@ -0,0 +1,18 @@
+use strict;
+use warnings;
+
+print "1..2\n";
+
+my @mapped_source = (1, 2);
+my @mapped = map {
+    unshift @mapped_source, 9;
+    $_;
+} @mapped_source;
+print join(',', @mapped) eq '1,2' ? "ok 1\n" : "not ok 1\n";
+
+my @grep_source = (1, 2);
+my @filtered = grep {
+    unshift @grep_source, 9;
+    1;
+} @grep_source;
+print join(',', @filtered) eq '1,2' ? "ok 2\n" : "not ok 2\n";

src/test/resources/unit/regex/callback_exception_unwind.t

@@ -0,0 +1,21 @@
+use strict;
+use warnings;
+
+our $localized = 'outer';
+'z' =~ /(z)/;
+
+print "1..4\n";
+{
+    local $^R = 9;
+    my $ok = eval {
+        'a' =~ /(a)(?{
+            local $localized = 'inside';
+            die "callback failure\n";
+        })/;
+        1;
+    };
+    print !defined($ok) && $@ =~ /callback failure/ ? "ok 1\n" : "not ok 1\n";
+    print $localized eq 'outer' ? "ok 2\n" : "not ok 2\n";
+    print $^R == 9 ? "ok 3\n" : "not ok 3\n";
+    print !defined($1) ? "ok 4\n" : "not ok 4\n";
+}

src/test/resources/unit/regex_tied_interpolation_fetch.t

@@ -0,0 +1,20 @@
+use strict;
+use warnings;
+
+{
+    package TiedPattern;
+    our $fetches = 0;
+    sub TIESCALAR { bless [$_[1]], $_[0] }
+    sub FETCH { $fetches++; $_[0][0] }
+}
+
+print "1..2\n";
+
+tie my $pattern, 'TiedPattern', 'foo';
+$_ = 'foo foo';
+/$pattern foo/;
+print $TiedPattern::fetches == 1 ? "ok 1\n" : "not ok 1\n";
+
+$TiedPattern::fetches = 0;
+s/$pattern foo//;
+print $TiedPattern::fetches == 1 ? "ok 2\n" : "not ok 2\n";

src/test/resources/unit/scalar_handle_tty_errno.t

@@ -0,0 +1,11 @@
+use strict;
+use warnings;
+
+print "1..2\n";
+
+my $buffer = "text";
+open my $handle, '<', \$buffer or die $!;
+$! = 0;
+my $is_tty = -t $handle;
+print !defined($is_tty) ? "ok 1\n" : "not ok 1\n";
+print 0 + $! == 9 ? "ok 2\n" : "not ok 2\n";