Merge pull request #975 from fglock/feature/joni-optimistic-callback-condition

Execute optimistic regex callbacks in Joni

Author: fglock

docs/design/joni-callout-fork.md

@@ -135,6 +135,13 @@ The parser emits a `regexTemplate` operation whose ordered parts contain normal
 interpolation values and explicit `regexCallback` wrappers. Runtime template
 construction assigns callback IDs and produces the engine-facing skeleton.
 
+Optimistic callbacks use the same structured path. Standalone `(*{ code })`
+callbacks execute as zero-width BLOCK callouts and publish their result through
+`$^R`; `(?(*{ code })yes|no)` predicates execute as CONDITION callouts, select
+their branch from the result's truth, and leave `$^R` unchanged. The matcher
+publishes `$^N` from capture-close order rather than deriving it from `$+`, whose
+meaning is the highest-numbered defined capture.
+
 An interpolated coderef remains ordinary interpolation because only the parser
 can create a callback wrapper. Runtime strings containing Perl eval groups never
 become trusted callback skeletons. They retain the existing security checks and,
@@ -242,6 +249,8 @@ keep their established fold-to-pattern path.
 - Focused tests cover provisional captures, repeated execution after
   backtracking, `FAIL`, nested frames, handler exceptions, interruption, and
   exact-once reverse-order unwind.
+- Optimistic-callback tests cover standalone execution, conditional truth,
+  `$^R`, exact `$^N` capture-close order, `$+`, and alternative reachability.
 - The standalone JAR contains no `org/joni` or `org/jcodings` classes, contains
   both relocated trees, and includes all required notices.
 - An embedding smoke test can load stock Joni and PerlOnJava together.

docs/reference/feature-matrix.md

@@ -394,10 +394,10 @@ my @copy = @{$z};         # ERROR
 - ❌  **Lookbehind Assertions**: Variable-length negative or positive lookbehind assertions, e.g., `(?<=...)` or `(?<!...)`, are not supported.
 - ✅  **Branch Reset Groups**: `(?|...)` resets capture numbering across alternatives and preserves mapped match variables.
 - ✅  **Advanced Subroutine Calls**: Sub-pattern calls with numbered or named references like `(?1)` and `(?&name)` execute through Joni.
-- 🟡  **Conditional Expressions**: Executable callback conditions `(?(?{ code })yes|no)` execute through Joni; other condition forms are not yet complete.
+- 🟡  **Conditional Expressions**: Executable callback conditions `(?(?{ code })yes|no)` and optimistic predicates `(?(*{ code })yes|no)` execute through Joni; other condition forms are not yet complete.
 - ❌  **Extended Unicode Regex Features**: Some extended Unicode regex functionalities are not supported.
 - ❌  **Extended Grapheme Clusters**: Matching with `\X` for extended grapheme clusters is not supported.
-- 🟡  **Embedded Code in Regex**: `(?{ code })`, executable callback conditions, and `(??{ code })` run as lexical closures in Joni with provisional captures and backtracking unwind. Optimistic callbacks `(*{ code })` are not supported.
+- ✅  **Embedded Code in Regex**: `(?{ code })`, optimistic callbacks `(*{ code })`, executable callback conditions, and `(??{ code })` run as lexical closures in Joni with provisional captures and backtracking unwind. `$^N` follows capture-close order independently of `$+`.
 - ✅  **Regex Debugging**: Lexically scoped `use/no re 'debug'` and `debugcolor` are supported, including runtime snapshot ownership.
 - 🟡  **Runtime Regex Evaluation**: `use re 'eval'` controls whether interpolated patterns containing eval groups may compile, but runtime strings cannot manufacture trusted callback closures.
 - ❌  **Regex Compilation Flags**: Setting default regex flags with `use re '/flags';` is not supported.

src/main/java/org/perlonjava/frontend/parser/StringSegmentParser.java

@@ -804,6 +804,9 @@ protected boolean handleSpecialToken(String text) {
                 } else if (isRegex && regexCodeBlocksAreActive() && isRegexRecursiveBlock()) {
                     parseRegexCodeBlock(true);   // (??{...}) - recursive pattern
                     yield true;
+                } else if (isRegex && regexCodeBlocksAreActive() && isRegexOptimisticBlock()) {
+                    parseRegexOptimisticBlock(); // (*{...}) - optimization-preserving callback
+                    yield true;
                 }
                 yield false;
             }
@@ -835,7 +838,8 @@ private boolean isRegexCallbackCondition() {
         return currentPos + 3 < parser.tokens.size()
                 && "?".equals(parser.tokens.get(currentPos).text)
                 && "(".equals(parser.tokens.get(currentPos + 1).text)
-                && "?".equals(parser.tokens.get(currentPos + 2).text)
+                && ("?".equals(parser.tokens.get(currentPos + 2).text)
+                    || "*".equals(parser.tokens.get(currentPos + 2).text))
                 && "{".equals(parser.tokens.get(currentPos + 3).text);
     }
 
@@ -844,7 +848,10 @@ private void parseRegexCallbackCondition() {
         int start = tokenIndex;
         TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?");
         TokenUtils.consume(parser, LexerTokenType.OPERATOR, "(");
-        TokenUtils.consume(parser, LexerTokenType.OPERATOR, "?");
+        LexerToken callbackType = TokenUtils.consume(parser);
+        if (!"?".equals(callbackType.text) && !"*".equals(callbackType.text)) {
+            throw new IllegalStateException("invalid regex callback condition marker");
+        }
         TokenUtils.consume(parser, LexerTokenType.OPERATOR, "{");
         Node block = parseBlock(parser);
         TokenUtils.consume(parser, LexerTokenType.OPERATOR, "}");
@@ -853,6 +860,24 @@ private void parseRegexCallbackCondition() {
         segments.add(regexCallback(block, "CONDITION", start));
     }
 
+    private boolean isRegexOptimisticBlock() {
+        int currentPos = parser.tokenIndex;
+        return currentPos + 1 < parser.tokens.size()
+                && "*".equals(parser.tokens.get(currentPos).text)
+                && "{".equals(parser.tokens.get(currentPos + 1).text);
+    }
+
+    private void parseRegexOptimisticBlock() {
+        flushCurrentSegment();
+        int start = tokenIndex;
+        TokenUtils.consume(parser, LexerTokenType.OPERATOR, "*");
+        TokenUtils.consume(parser, LexerTokenType.OPERATOR, "{");
+        Node block = parseBlock(parser);
+        TokenUtils.consume(parser, LexerTokenType.OPERATOR, "}");
+        TokenUtils.consume(parser, LexerTokenType.OPERATOR, ")");
+        segments.add(regexCallback(block, "BLOCK", start));
+    }
+
     /**
      * Checks if the current tokens form a (??{...}) recursive regex pattern.
      * This is similar to (?{...}) but uses the result as a regex pattern.

src/main/java/org/perlonjava/runtime/regex/JoniRegexPattern.java

@@ -421,6 +421,7 @@ public String group(String name) {
         }
 
         @Override public int groupCount() { return regex.numberOfCaptures(); }
+        @Override public int lastClosedCapture() { return matcher.lastClosedCapture(); }
         @Override public Map<String, Integer> namedGroups() { return namedGroups; }
         @Override public String patternDescription() { return sourcePattern; }
 
@@ -649,6 +650,9 @@ private void publishProvisional(MatchView match) {
                     state.lastCaptureGroups[group - 1] = input.substring(begin, end);
                 }
             }
+            int lastClosed = match.lastClosedCapture();
+            state.lastClosedCapture = lastClosed > 0 && lastClosed <= count
+                    ? state.lastCaptureGroups[lastClosed - 1] : null;
         }
 
         private int charOffset(int byteOffset) {

src/main/java/org/perlonjava/runtime/regex/RegexMatcher.java

@@ -32,6 +32,21 @@ public interface RegexMatcher {
 
     int groupCount();
 
+    /** Most recently closed capture number, or -1 when the backend cannot provide it. */
+    default int lastClosedCapture() {
+        int selected = -1;
+        int selectedEnd = -1;
+        for (int group = 1; group <= groupCount(); group++) {
+            int end = end(group);
+            if (end > selectedEnd || (end == selectedEnd && end >= 0
+                    && (selected < 0 || group < selected))) {
+                selected = group;
+                selectedEnd = end;
+            }
+        }
+        return selected;
+    }
+
     Map<String, Integer> namedGroups();
 
     String patternDescription();

src/main/java/org/perlonjava/runtime/regex/RuntimeRegex.java

@@ -1543,6 +1543,9 @@ private static void updateNumberedCaptureGroups(RuntimeRegex regex, RegexMatcher
         regexState.manualCaptureStarts = null;
         regexState.manualCaptureEnds = null;
         int captureCount = matcher.groupCount();
+        int lastClosedCapture = matcher.lastClosedCapture();
+        regexState.lastClosedCapture = lastClosedCapture > 0
+                && lastClosedCapture <= captureCount ? matcher.group(lastClosedCapture) : null;
         if (captureCount == 0) {
             regexState.lastCaptureGroups = null;
             return;

src/main/java/org/perlonjava/runtime/runtimetypes/GlobalContext.java

@@ -75,8 +75,8 @@ public static void initializeGlobals(CompilerOptions compilerOptions) {
             String varName = "main::" + Character.toString(c - 'A' + 1);
             GlobalVariable.getGlobalVariable(varName);
         }
-        // $^N - last capture group closed (not yet implemented, but must be read-only)
-        GlobalVariable.globalVariables.put(encodeSpecialVar("N"), new RuntimeScalarReadOnly());
+        GlobalVariable.globalVariables.put(encodeSpecialVar("N"),
+                new ScalarSpecialVariable(ScalarSpecialVariable.Id.LAST_CLOSED_PAREN_MATCH));
         // $^S - current state of the interpreter (undef=compiling, 0=not in eval, 1=in eval)
         GlobalVariable.globalVariables.put("main::" + Character.toString('S' - 'A' + 1),
                 new ScalarSpecialVariable(ScalarSpecialVariable.Id.EVAL_STATE));

src/main/java/org/perlonjava/runtime/runtimetypes/RegexState.java

@@ -22,6 +22,7 @@ public class RegexState implements DynamicState {
     private final boolean lastMatchUsedPFlag;
     private final boolean lastMatchUsedBackslashK;
     private final String[] lastCaptureGroups;
+    private final String lastClosedCapture;
     private final Map<String, List<String>> lastNamedCaptureGroups;
     private final boolean lastMatchWasByteString;
     private final boolean lastMatchResultsTainted;
@@ -44,6 +45,7 @@ public RegexState() {
         lastMatchUsedPFlag = state.lastMatchUsedPFlag;
         lastMatchUsedBackslashK = state.lastMatchUsedBackslashK;
         lastCaptureGroups = state.lastCaptureGroups;
+        lastClosedCapture = state.lastClosedCapture;
         lastNamedCaptureGroups = state.lastNamedCaptureGroups;
         lastMatchWasByteString = state.lastMatchWasByteString;
         lastMatchResultsTainted = state.lastMatchResultsTainted;
@@ -82,6 +84,7 @@ public void dynamicRestoreState() {
         state.lastMatchUsedPFlag = lastMatchUsedPFlag;
         state.lastMatchUsedBackslashK = lastMatchUsedBackslashK;
         state.lastCaptureGroups = lastCaptureGroups;
+        state.lastClosedCapture = lastClosedCapture;
         state.lastNamedCaptureGroups = lastNamedCaptureGroups;
         state.lastMatchWasByteString = lastMatchWasByteString;
         state.lastMatchResultsTainted = lastMatchResultsTainted;

src/main/java/org/perlonjava/runtime/runtimetypes/RuntimeRegexState.java

@@ -32,6 +32,7 @@ public final class RuntimeRegexState {
     public boolean lastMatchUsedPFlag;
     public boolean lastMatchUsedBackslashK;
     public String[] lastCaptureGroups;
+    public String lastClosedCapture;
     public Map<String, List<String>> lastNamedCaptureGroups;
     public boolean lastMatchWasByteString;
     public boolean lastMatchResultsTainted;
@@ -85,6 +86,7 @@ public void clearMatchState() {
         lastMatchUsedPFlag = false;
         lastMatchUsedBackslashK = false;
         lastCaptureGroups = null;
+        lastClosedCapture = null;
         lastNamedCaptureGroups = null;
         lastMatchWasByteString = false;
         lastMatchResultsTainted = false;

src/main/java/org/perlonjava/runtime/runtimetypes/ScalarSpecialVariable.java

@@ -227,6 +227,10 @@ public RuntimeScalar getValueAsScalar() {
                     String lastCapture = RuntimeRegex.lastCaptureString();
                     yield lastCapture != null ? makeRegexResultScalar(lastCapture) : scalarUndef;
                 }
+                case LAST_CLOSED_PAREN_MATCH -> {
+                    String lastCapture = PerlRuntime.current().regexState.lastClosedCapture;
+                    yield lastCapture != null ? makeRegexResultScalar(lastCapture) : scalarUndef;
+                }
                 case LAST_SUCCESSFUL_PATTERN -> PerlRuntime.current().regexState.lastSuccessfulPattern != null
                         ? new RuntimeScalar(PerlRuntime.current().regexState.lastSuccessfulPattern) : scalarUndef;
                 case LAST_REGEXP_CODE_RESULT -> {
@@ -527,6 +531,7 @@ public enum Id {
         LAST_FH,    // Represents the last filehandle used in an input operation.
         INPUT_LINE_NUMBER, // Represents the current line number in an input operation.
         LAST_PAREN_MATCH, // The highest capture variable ($1, $2, ...) which has a defined value.
+        LAST_CLOSED_PAREN_MATCH, // $^N - most recently closed capture in the current match.
         LAST_SUCCESSFUL_PATTERN, // ${^LAST_SUCCESSFUL_PATTERN}
         LAST_REGEXP_CODE_RESULT, // $^R - Result of last (?{...}) code block in regex
         HINTS, // $^H - Compile-time hints (strict, etc.)