Merge pull request #977 from fglock/fix/cpan-tooling-perl6-greple

Fix reusable CPAN compiler and process tooling

Author: fglock

dev/modules/cpan_tooling_perl6_greple.md

@@ -0,0 +1,89 @@
+# CPAN compiler/tooling batch: Perl::ToPerl6 through Devel::SlowBless
+
+## Goal
+
+Classify and repair the reusable compiler, regex, runtime, and CPAN-tooling
+defects exposed by `jcpan -t` for Perl::ToPerl6,
+Game::HeroesVsAliens::Alien, App::Greple::update, GCJ::Cni,
+Data::Semantic, Text::HTML::CollapseWhitespace, DBM::Deep::Blue, and
+Devel::SlowBless. Distributions that fail under the same local system Perl are
+out of scope, as requested.
+
+## System Perl classification (2026-08-16)
+
+| Target | Classification |
+|---|---|
+| Perl::ToPerl6 | Ignore: system Perl also fails `t/05_utils.t` test 136. |
+| Game::HeroesVsAliens::Alien | Ignore: system Perl cannot build Alien::SDL/SDL; the native dependency download/build fails. |
+| App::Greple::update | In scope: system Perl passes 2 files / 4 tests. |
+| GCJ::Cni | Ignore: system Perl cannot find the obsolete `gcj/cni.h` toolchain. |
+| Data::Semantic | In scope: system Perl passes 15 files / 16 tests. |
+| Text::HTML::CollapseWhitespace | Ignore: its Text-HTML-Turndown distribution also fails under system Perl (`t/01-turndown.t` test 46). |
+| DBM::Deep::Blue | Ignore: its native source includes unavailable `malloc.h` and fails compilation on this host. |
+| Devel::SlowBless | In scope: system Perl passes 1 file / 4 tests. |
+
+Full baseline output is captured in `/tmp/jcpan_*_baseline.log` and
+`/tmp/system_perl_*` logs.
+
+## Progress Tracking
+
+### Current Status: Complete (2026-08-16)
+
+### Completed Phases
+
+- [x] Phase 1: Baseline and system-Perl classification (2026-08-16)
+  - Ran every requested `jcpan -t` target sequentially with hard timeouts.
+  - Installed missing system-Perl prerequisites into an isolated `/tmp` tree
+    and tested the exact unpacked distributions.
+- [x] Phase 2: Root-cause reduction (2026-08-16)
+  - App::Greple::update requires named-capture regex conditionals, which the
+    vendored Joni engine supports but the backend router did not select.
+  - Data::Semantic exposes a lazy generated-sub VerifyError after module
+    registration side effects; file-level interpreter retry then duplicates
+    those registrations.
+  - Devel::SlowBless needs its two XS generation-counter functions backed by
+    runtime-owned Java state.
+- [x] Phase 3: Implementation (2026-08-16)
+  - Routed named-capture conditionals to Joni.
+  - Translated Perl brace-form named backreferences for Joni and covered
+    recursive named patterns.
+  - Added eager lazy-sub verification so only the invalid sub falls back.
+  - Added the Devel::SlowBless Java XS bridge and runtime sub-generation
+    counter.
+  - Corrected string typeglob aliases and ampersand calls inside
+    `delete`/`exists` subscripts.
+  - Implemented registered-child handling for `wait`/`waitpid`, made pipe
+    output pumps drain before close returns, and isolated replay-process STDIN
+    until the fork point. These fixes make Perl's `open HANDLE, '|-'` filter
+    pattern reliable without distribution-specific changes.
+  - Added system-Perl-validated regression tests.
+- [x] Phase 4: End-to-end validation (2026-08-16)
+  - `App::Greple::update`: 2 files / 4 tests, PASS.
+  - `Data::Semantic`: 15 files / 16 tests, PASS.
+  - `Devel::SlowBless`: bundled upstream suite, 1 file / 4 tests, PASS.
+  - All new unit tests pass under system Perl and both PerlOnJava backends.
+  - Full `make` completed successfully.
+
+### Files Changed
+
+- Compiler/backend: `EmitSubroutine.java`, `ParseInfix.java`.
+- Regex: `JoniRegexPattern.java`.
+- Process and IO runtime: `PerlRuntime.java`, `RuntimeIO.java`,
+  `PipeOutputChannel.java`, `WaitpidOperator.java`.
+- Runtime semantics: `RuntimeGlob.java`, `MroRuntimeState.java`, `Mro.java`.
+- Java module bridge: `DevelSlowBless.java`, `Devel/SlowBless.pm`.
+- Regression coverage: eight focused tests in `src/test/resources/unit`.
+
+### Next Steps
+
+1. Merge after CI and review.
+
+### Open Questions
+
+- None. Native distributions excluded by the system-Perl rule remain
+  documented rather than hidden behind distribution preferences.
+
+## Related References
+
+- `.agents/skills/debug-perlonjava/SKILL.md`
+- `docs/guides/module-porting.md`

src/main/java/org/perlonjava/backend/jvm/EmitSubroutine.java

@@ -14,6 +14,7 @@
 import org.perlonjava.runtime.runtimetypes.NameNormalizer;
 import org.perlonjava.runtime.runtimetypes.GlobalVariable;
 import org.perlonjava.runtime.runtimetypes.PerlCompilerException;
+import org.perlonjava.runtime.runtimetypes.RuntimeArray;
 import org.perlonjava.runtime.runtimetypes.RuntimeBase;
 import org.perlonjava.runtime.runtimetypes.RuntimeCode;
 import org.perlonjava.runtime.runtimetypes.RuntimeContextType;
@@ -262,6 +263,22 @@ public static void emitSubroutine(EmitterContext ctx, SubroutineNode node) {
                     EmitterMethodCreator.createClassWithMethod(
                             subCtx, node.block, node.useTryCatch
                     );
+            try {
+                // HotSpot can defer verification of a generated lazy sub until
+                // it is first invoked. If that happens after the enclosing file
+                // body has made writes, a file-level retry replays those writes.
+                // Resolve apply() now so only the invalid sub falls back.
+                generatedClass.getDeclaredMethod(
+                        "apply", RuntimeArray.class, int.class);
+            } catch (VerifyError | ClassFormatError verificationFailure) {
+                InterpretedCode interpreted = EmitterMethodCreator.compileToInterpreter(
+                        node.block, subCtx, node.useTryCatch);
+                throw new InterpreterFallbackException(interpreted, newEnv);
+            } catch (NoSuchMethodException reflectionFailure) {
+                throw new PerlCompilerException(
+                        "Failed to resolve generated subroutine: "
+                                + reflectionFailure.getMessage());
+            }
             String newClassNameDot = subCtx.javaClassInfo.javaClassName.replace('/', '.');
             if (CompilerOptions.DEBUG_ENABLED) ctx.logDebug("Generated class name: " + newClassNameDot + " internal " + subCtx.javaClassInfo.javaClassName);
             if (CompilerOptions.DEBUG_ENABLED) ctx.logDebug("Generated class env:  " + Arrays.toString(newEnv));

src/main/java/org/perlonjava/frontend/parser/ParseInfix.java

@@ -522,7 +522,17 @@ private static List<Node> parseArraySubscript(Parser parser) {
         // backtrack
         parser.tokenIndex = currentIndex;
 
-        return ListParser.parseList(parser, "]", 1);
+        // A subscript is an ordinary expression context even when its outer
+        // operator is delete/exists/defined (or reference-taking syntax).
+        // In particular, delete $array[&index] must invoke &index with the
+        // caller's @_, not turn it into a CODE value.
+        boolean savedParsingTakeReference = parser.parsingTakeReference;
+        parser.parsingTakeReference = false;
+        try {
+            return ListParser.parseList(parser, "]", 1);
+        } finally {
+            parser.parsingTakeReference = savedParsingTakeReference;
+        }
 
     }
 
@@ -573,7 +583,17 @@ static List<Node> parseHashSubscript(Parser parser) {
         // backtrack
         parser.tokenIndex = currentIndex;
 
-        return ListParser.parseList(parser, "}", 1);
+        // The reference-taking mode used while parsing delete/exists applies
+        // to their direct operand, not to expressions inside a subscript.
+        // Getopt::EX relies on delete $hash{&CONSTANT_SUB} calling the sub and
+        // using its return value as the key.
+        boolean savedParsingTakeReference = parser.parsingTakeReference;
+        parser.parsingTakeReference = false;
+        try {
+            return ListParser.parseList(parser, "}", 1);
+        } finally {
+            parser.parsingTakeReference = savedParsingTakeReference;
+        }
     }
 
     /**

src/main/java/org/perlonjava/runtime/io/PipeOutputChannel.java

@@ -81,6 +81,13 @@ public ThreadInheritancePolicy threadInheritancePolicy() {
      */
     private BufferedReader outputReader;
 
+    /** Pumps must finish before close() returns or callers can restore STDOUT
+     * and lose the final child output. */
+    private Thread outputThread;
+    private Thread errorThread;
+    private RuntimeIO outputDestination;
+    private RuntimeIO errorDestination;
+
     /**
      * Tracks whether the pipe has been closed
      */
@@ -180,6 +187,15 @@ private void setupProcess(ProcessBuilder processBuilder) throws IOException {
 
     private void setupProcess(ProcessBuilder processBuilder, Map<String, String> environmentOverrides)
             throws IOException {
+        // Capture the destinations before IOOperator installs this pipe into
+        // the target glob.  For `open STDOUT, '|-'`, looking up STDOUT from the
+        // pump thread later would find this same pipe and feed child output
+        // back into its stdin instead of forwarding it to the saved stdout.
+        outputDestination = borrowedDestination(
+                GlobalVariable.getGlobalIO("main::STDOUT").getRuntimeIO());
+        errorDestination = borrowedDestination(
+                GlobalVariable.getGlobalIO("main::STDERR").getRuntimeIO());
+
         // Set working directory to current directory
         String userDir = org.perlonjava.runtime.runtimetypes.RuntimeEnvironment.currentDirectory();
         processBuilder.directory(new File(userDir));
@@ -201,13 +217,13 @@ private void setupProcess(ProcessBuilder processBuilder, Map<String, String> env
         // Start threads to consume stdout and stderr and route through Perl handles
         // This ensures Perl-level redirections are honored
         PerlRuntime runtime = PerlRuntime.current();
-        Thread outputThread = new Thread(() -> {
+        outputThread = new Thread(() -> {
             try (PerlRuntime.Binding ignored = runtime.bind();
                  BufferedReader out = outputReader) {
                 String line;
                 while ((line = out.readLine()) != null) {
                     try {
-                        RuntimeIO perlStdout = GlobalVariable.getGlobalIO("main::STDOUT").getRuntimeIO();
+                        RuntimeIO perlStdout = outputDestination;
                         if (perlStdout != null) {
                             perlStdout.write(line + "\n");
                         } else {
@@ -224,13 +240,13 @@ private void setupProcess(ProcessBuilder processBuilder, Map<String, String> env
         outputThread.setDaemon(true);
         outputThread.start();
 
-        Thread errorThread = new Thread(() -> {
+        errorThread = new Thread(() -> {
             try (PerlRuntime.Binding ignored = runtime.bind();
                  BufferedReader err = errorReader) {
                 String line;
                 while ((line = err.readLine()) != null) {
                     try {
-                        RuntimeIO perlStderr = GlobalVariable.getGlobalIO("main::STDERR").getRuntimeIO();
+                        RuntimeIO perlStderr = errorDestination;
                         if (perlStderr != null) {
                             perlStderr.write(line + "\n");
                         } else {
@@ -248,6 +264,13 @@ private void setupProcess(ProcessBuilder processBuilder, Map<String, String> env
         errorThread.start();
     }
 
+    private static RuntimeIO borrowedDestination(RuntimeIO destination) {
+        if (destination == null) return null;
+        RuntimeIO borrowed = new RuntimeIO(destination.ioHandle);
+        borrowed.globName = destination.globName;
+        return borrowed;
+    }
+
     /**
      * Writes a string to the process stdin.
      *
@@ -334,6 +357,8 @@ public RuntimeScalar close() {
                     exitCode = -1;
                 }
             }
+            joinPump(outputThread);
+            joinPump(errorThread);
             getGlobalVariable("main::?").set(exitCode << 8);
 
             isClosed = true;
@@ -343,6 +368,15 @@ public RuntimeScalar close() {
         }
     }
 
+    private static void joinPump(Thread pump) {
+        if (pump == null || pump == Thread.currentThread()) return;
+        try {
+            pump.join();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
     /**
      * EOF for output pipes - always true since you can't read from write-only pipes.
      *

src/main/java/org/perlonjava/runtime/mro/MroRuntimeState.java

@@ -31,6 +31,7 @@ public final class MroRuntimeState {
     private InheritanceResolver.MROAlgorithm defaultMro = InheritanceResolver.MROAlgorithm.DFS;
     private boolean autoloadEnabled = true;
     private long isaGeneration;
+    private long subGeneration = 1;
     private long isaRevGeneration = -1;
     private long observedSymbolMutationEpoch;
     private long observedIsaMutationEpoch;
@@ -87,6 +88,14 @@ public long isaGeneration() {
         return isaGeneration;
     }
 
+    public long subGeneration() {
+        return subGeneration;
+    }
+
+    public void incrementSubGeneration() {
+        subGeneration++;
+    }
+
     public long isaRevGeneration() {
         return isaRevGeneration;
     }

src/main/java/org/perlonjava/runtime/operators/WaitpidOperator.java

@@ -67,6 +67,11 @@ private static RuntimeScalar waitpidPosix(int pid, int flags) {
             if (javaProcess != null) {
                 return waitpidJavaProcess(pid, javaProcess, flags);
             }
+        } else {
+            RuntimeScalar javaChildResult = waitForAnyJavaProcess(flags);
+            if (javaChildResult != null) {
+                return javaChildResult;
+            }
         }
         try {
             int[] status = new int[1];
@@ -90,6 +95,38 @@ private static RuntimeScalar waitpidPosix(int pid, int flags) {
         }
     }
 
+    /**
+     * Wait for a child created by pipe open or another Java ProcessBuilder path.
+     * On POSIX the JDK has its own native reaper, so calling libc waitpid(-1)
+     * after reading a pipe can report ECHILD even though PerlOnJava still owns
+     * an unreaped logical child.  Prefer the registered Process objects before
+     * falling through to native children.
+     */
+    private static RuntimeScalar waitForAnyJavaProcess(int flags) {
+        Map<Long, Process> children = RuntimeIO.childProcessesSnapshot();
+        if (children.isEmpty()) return null;
+
+        boolean nonBlocking = (flags & WNOHANG) != 0;
+        for (Map.Entry<Long, Process> entry : children.entrySet()) {
+            if (!entry.getValue().isAlive()) {
+                return waitpidJavaProcess(entry.getKey().intValue(), entry.getValue(), flags);
+            }
+        }
+        if (nonBlocking) return new RuntimeScalar(0);
+
+        java.util.concurrent.CompletableFuture<?>[] exits = children.values().stream()
+                .map(Process::onExit)
+                .toArray(java.util.concurrent.CompletableFuture[]::new);
+        java.util.concurrent.CompletableFuture.anyOf(exits).join();
+
+        for (Map.Entry<Long, Process> entry : RuntimeIO.childProcessesSnapshot().entrySet()) {
+            if (!entry.getValue().isAlive()) {
+                return waitpidJavaProcess(entry.getKey().intValue(), entry.getValue(), flags);
+            }
+        }
+        return new RuntimeScalar(-1);
+    }
+
     private static RuntimeScalar waitpidJavaProcess(int pid, Process process, int flags) {
         boolean nonBlocking = (flags & WNOHANG) != 0;
         boolean chldIgnore = isChldIgnored();

src/main/java/org/perlonjava/runtime/perlmodule/DevelSlowBless.java

@@ -0,0 +1,36 @@
+package org.perlonjava.runtime.perlmodule;
+
+import org.perlonjava.runtime.runtimetypes.PerlRuntime;
+import org.perlonjava.runtime.runtimetypes.RuntimeArray;
+import org.perlonjava.runtime.runtimetypes.RuntimeList;
+import org.perlonjava.runtime.runtimetypes.RuntimeScalar;
+
+/** Java XS implementation of Devel::SlowBless's generation counters. */
+public class DevelSlowBless extends PerlModuleBase {
+
+    public static final String XS_VERSION = "0.06";
+
+    public DevelSlowBless() {
+        super("Devel::SlowBless", false);
+    }
+
+    public static void initialize() {
+        DevelSlowBless module = new DevelSlowBless();
+        try {
+            module.registerMethod("sub_gen", null);
+            module.registerMethod("amg_gen", null);
+        } catch (NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static RuntimeList sub_gen(RuntimeArray args, int ctx) {
+        return new RuntimeScalar(PerlRuntime.current().mroState().subGeneration()).getList();
+    }
+
+    public static RuntimeList amg_gen(RuntimeArray args, int ctx) {
+        // PL_amagic_generation was removed from Perl in 5.17.1. The XS
+        // distribution returns zero on all modern Perls.
+        return new RuntimeScalar(0).getList();
+    }
+}

src/main/java/org/perlonjava/runtime/perlmodule/Mro.java

@@ -443,9 +443,10 @@ public static RuntimeList get_pkg_gen(RuntimeArray args, int ctx) {
      * @param packageName The name of the package.
      */
     public static void incrementPackageGeneration(String packageName) {
-        Map<String, Integer> packageGenerations =
-                PerlRuntime.current().mroState().packageGenerations();
+        var state = PerlRuntime.current().mroState();
+        Map<String, Integer> packageGenerations = state.packageGenerations();
         Integer current = packageGenerations.getOrDefault(packageName, 1);
         packageGenerations.put(packageName, current + 1);
+        state.incrementSubGeneration();
     }
 }