Provide information about build progress

README.md

@@ -4,6 +4,7 @@ Extender is a build server that builds native extensions of the Defold engine. T
 
 * Server description and setup/run instructions - [link](/server/README.md)
 * Debugging FAQ - [link](/README_DEBUGGING.md)
+* Live build progress (SSE) - [link](/README_BUILD_PROGRESS.md)
 
 ---
 

README_BUILD_PROGRESS.md

@@ -0,0 +1,106 @@
+# Live build progress
+
+The extender streams live build progress over Server-Sent Events (SSE) so clients can show
+what a build is doing (downloading the SDK, resolving dependencies, compiling file N of M,
+linking, packaging) instead of a silent wait.
+
+Progress is **advisory**: `/job_status` polling remains the source of truth for build
+completion, and terminal progress events are only emitted after the result files
+(`build.zip`/`error.txt`) are in place. Everything is backward compatible — old clients
+never call the endpoint, and new clients fall back to polling when the server has no
+progress support.
+
+## Endpoint
+
+```
+GET /job_progress?jobId=<jobId>
+Accept: text/event-stream
+```
+
+* Live job: streams `progress` events. A snapshot of the current state is sent immediately
+  on subscribe (early events always precede the first subscriber, because the build is
+  dispatched before the jobId is returned).
+* Reconnects can pass the standard `Last-Event-ID` header; missed events are replayed from
+  a per-job ring buffer when possible, otherwise a state snapshot is sent.
+* Job already finished (result files on disk): a single terminal event, then the stream closes.
+* Unknown job, or `extender.progress.enabled: false`: `404`.
+* Comment lines (`:ka`) are heartbeats sent every `heartbeat-interval` to keep idle
+  connections alive through proxies.
+
+Event payload (JSON, `id:` field = `seq`):
+
+```json
+{
+  "jobId": "job1234567890",
+  "seq": 17,
+  "ts": 1720512345678,
+  "stage": "COMPILING",
+  "detail": "extension1: compiling source files",
+  "percent": 55,
+  "extension": "extension1",
+  "currentFile": 12,
+  "totalFiles": 34,
+  "terminal": false
+}
+```
+
+Stages, in pipeline order: `RECEIVED`, `QUEUED`, `SDK`, `DEPENDENCIES`, `MANIFESTS`,
+`PLATFORM`, `COMPILING`, `LINKING`, `PACKAGING`, and the terminals `SUCCESS`/`ERROR`.
+`REMOTE_BUILDING` is a coarse stage used by a frontend when its remote builder runs an
+older server without progress support. `percent` is a 0-100 estimate and never decreases.
+`extension`/`currentFile`/`totalFiles` are only present while compiling.
+
+## Frontend / remote builder setups
+
+A frontend instance relays the remote builder's progress stream to its own subscribers
+under the frontend's jobId, so external clients only ever talk to the frontend. If the
+remote builder runs an older server version the frontend degrades to coarse
+`REMOTE_BUILDING` ticks.
+
+If the server sits behind a buffering reverse proxy, response buffering must be disabled
+for `/job_progress` (e.g. nginx `proxy_buffering off` or the `X-Accel-Buffering: no`
+response header), otherwise events arrive in bursts or not at all. The built-in heartbeats
+keep idle timeouts (Jetty and load balancers) from closing quiet streams.
+
+## Server configuration (`application.yml`)
+
+```yaml
+extender:
+    progress:
+        enabled: true               # false restores the old behavior exactly
+        sse-timeout: 1800000        # SseEmitter timeout, ms
+        heartbeat-interval: 15000   # keepalive comment cadence, ms
+        event-buffer-size: 256      # per-job replay buffer for Last-Event-ID
+        max-subscribers-per-job: 8
+        registry-ttl: 1200000       # sweep for jobs that died without a terminal event
+        cleanup-period: 20000
+```
+
+## Client API
+
+`ExtenderClient` gained an overload that reports progress while the existing polling flow
+runs unchanged:
+
+```java
+extenderClient.build(platform, sdkVersion, sourceResources, destination, log,
+    (stage, detail, percent, currentFile, totalFiles) -> {
+        // called on a background thread; currentFile/totalFiles are -1 outside COMPILING
+        System.out.printf("[%3d%%] %s %s%n", percent, stage, detail);
+    });
+```
+
+System properties:
+
+* `com.defold.extender.client.progress-enabled` (default `true`) — set `false` to never
+  open the progress stream.
+* `com.defold.extender.client.progress-reconnect-attempts` (default `5`) — reconnect
+  attempts for a dropped stream.
+
+## Trying it with curl
+
+```sh
+JOB=$(curl -s -X POST -F "file=@upload.zip" http://localhost:9000/build_async/x86_64-linux/<sdk-sha1>)
+curl -sN "http://localhost:9000/job_progress?jobId=$JOB"
+# reconnect mid-build and replay everything after event 10:
+curl -sN -H "Last-Event-ID: 10" "http://localhost:9000/job_progress?jobId=$JOB"
+```

README_CLIENT.md

@@ -12,4 +12,7 @@ There is a client part of the Extender code which is used in Bob.jar.
 
     $ cp -v ./build/libs/extender-client-0.0.1.jar <defold>/com.dynamo.cr/com.dynamo.cr.common/ext/extender-client-0.0.1.jar
 
+The client can report live build progress through an `ExtenderProgressListener` -
+see [README_BUILD_PROGRESS.md](/README_BUILD_PROGRESS.md).
+
 

client/src/main/java/com/defold/extender/client/ExtenderClient.java

@@ -58,6 +58,7 @@ public class ExtenderClient {
     private ExtenderClientCache cache;
     private long buildSleepTimeout;
     private long buildResultWaitTimeout;
+    private boolean progressEnabled;
     private List<BasicHeader> headers;
     private HttpClient httpClient;
 
@@ -97,6 +98,7 @@ public ExtenderClient(ExtenderClientCache cache,
         this.cache = cache;
         this.buildSleepTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-sleep-timeout", "5000"));
         this.buildResultWaitTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-wait-timeout", "1200000"));
+        this.progressEnabled = Boolean.parseBoolean(System.getProperty("com.defold.extender.client.progress-enabled", "true"));
         this.headers = new ArrayList<BasicHeader>();
         this.httpClient = httpClient;
     }
@@ -238,7 +240,8 @@ static Set<String> getCachedFiles(String json) throws ExtenderClientException {
     }
 
 
-    private void build_async(String platform, String sdkVersion, HttpEntity entity, File destination, File log) throws ExtenderClientException {
+    private void build_async(String platform, String sdkVersion, HttpEntity entity, File destination, File log, ExtenderProgressListener progressListener) throws ExtenderClientException {
+        ExtenderProgressConsumer progressConsumer = null;
         try {
             String url = String.format("%s/build_async/%s/%s", extenderBaseUrl, platform, sdkVersion);
             HttpPost request = createPostRequest(url);
@@ -254,6 +257,15 @@ private void build_async(String platform, String sdkVersion, HttpEntity entity,
                 String jobId = EntityUtils.toString(response.getEntity());
                 String traceId = response.getFirstHeader(TRACE_ID_HEADER_NAME).getValue();
                 log("Async build request was accepted as job %s (traceId: %s)", jobId, traceId == null ? "null" : traceId);
+                if (progressListener != null && progressEnabled) {
+                    // advisory SSE progress stream; the poll loop below stays
+                    // the sole authority on build completion
+                    progressConsumer = new ExtenderProgressConsumer(httpClient, extenderBaseUrl, jobId,
+                            this::createGetRequest, progressListener);
+                    Thread progressThread = new Thread(progressConsumer, "extender-progress-" + jobId);
+                    progressThread.setDaemon(true);
+                    progressThread.start();
+                }
                 long currentTime = System.currentTimeMillis();
                 Integer jobStatus = 0;
                 Thread.sleep(buildSleepTimeout);
@@ -312,6 +324,10 @@ private void build_async(String platform, String sdkVersion, HttpEntity entity,
         }
         catch (Exception e) {
             throw new ExtenderClientException("Failed to communicate with Extender service.", e);
+        } finally {
+            if (progressConsumer != null) {
+                progressConsumer.stop();
+            }
         }
     }
 
@@ -362,6 +378,18 @@ HttpEntity createBuildRequestPayload(List<ExtenderResource> sourceResources) thr
      * @throws ExtenderClientException
      */
     public void build(String platform, String sdkVersion, List<ExtenderResource> sourceResources, File destination, File log) throws ExtenderClientException {
+        build(platform, sdkVersion, sourceResources, destination, log, null);
+    }
+
+    /**
+     * Builds a new engine and reports live build progress.
+     *
+     * @param progressListener Receives progress updates on a background thread
+     *                         while the build runs, or null. Progress is advisory:
+     *                         it may stop arriving (old server, dropped connection)
+     *                         while the build continues.
+     */
+    public void build(String platform, String sdkVersion, List<ExtenderResource> sourceResources, File destination, File log, ExtenderProgressListener progressListener) throws ExtenderClientException {
         String cacheKey = cache.calcKey(platform, sdkVersion, sourceResources);
         boolean isCached = cache.isCached(platform, cacheKey);
         if (isCached) {
@@ -370,7 +398,7 @@ public void build(String platform, String sdkVersion, List<ExtenderResource> sou
         }
 
         HttpEntity payload = createBuildRequestPayload(sourceResources);
-        build_async(platform, sdkVersion, payload, destination, log);
+        build_async(platform, sdkVersion, payload, destination, log, progressListener);
 
         // Store the new build
         cache.put(platform, cacheKey, destination);

client/src/main/java/com/defold/extender/client/ExtenderProgressConsumer.java

@@ -0,0 +1,186 @@
+package com.defold.extender.client;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.util.EntityUtils;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Consumes the server's /job_progress SSE stream on a background thread
+ * and forwards events to an ExtenderProgressListener.
+ *
+ * Strictly advisory: any failure (404 from an old server, dropped
+ * connection, malformed data) is swallowed after bounded reconnect
+ * attempts. The poll loop in ExtenderClient.build_async remains the sole
+ * authority on build completion and calls stop() when the build is done.
+ */
+class ExtenderProgressConsumer implements Runnable {
+    private static final Logger logger = Logger.getLogger(ExtenderProgressConsumer.class.getName());
+
+    /** Creates GET requests carrying the client's auth and custom headers. */
+    interface GetRequestFactory {
+        HttpGet create(String url) throws IOException;
+    }
+
+    private static final long RECONNECT_BACKOFF_MS = 2000;
+    private static final String RECONNECT_ATTEMPTS_PROPERTY = "com.defold.extender.client.progress-reconnect-attempts";
+    private static final int DEFAULT_RECONNECT_ATTEMPTS = 5;
+
+    private final HttpClient httpClient;
+    private final String jobProgressUrl;
+    private final GetRequestFactory requestFactory;
+    private final ExtenderProgressListener listener;
+    private final int maxReconnectAttempts;
+
+    private volatile boolean stopped = false;
+    private volatile HttpGet currentRequest = null;
+    private long lastEventId = -1;
+
+    ExtenderProgressConsumer(HttpClient httpClient, String extenderBaseUrl, String jobId,
+                             GetRequestFactory requestFactory, ExtenderProgressListener listener) {
+        this.httpClient = httpClient;
+        this.jobProgressUrl = String.format("%s/job_progress?jobId=%s", extenderBaseUrl, jobId);
+        this.requestFactory = requestFactory;
+        this.listener = listener;
+        this.maxReconnectAttempts = resolveMaxReconnectAttempts();
+    }
+
+    /**
+     * Reads the reconnect-attempts override from a system property, falling back
+     * to the default when the property is absent or not a valid integer.
+     */
+    private static int resolveMaxReconnectAttempts() {
+        String value = System.getProperty(RECONNECT_ATTEMPTS_PROPERTY);
+        if (value == null) {
+            return DEFAULT_RECONNECT_ATTEMPTS;
+        }
+        try {
+            return Integer.parseInt(value.trim());
+        } catch (NumberFormatException e) {
+            logger.log(Level.WARNING, "Ignoring malformed {0} value ''{1}''; using default {2}",
+                    new Object[] { RECONNECT_ATTEMPTS_PROPERTY, value, DEFAULT_RECONNECT_ATTEMPTS });
+            return DEFAULT_RECONNECT_ATTEMPTS;
+        }
+    }
+
+    /** Stops the consumer and unblocks the stream read. Safe to call more than once. */
+    void stop() {
+        stopped = true;
+        HttpGet request = currentRequest;
+        if (request != null) {
+            request.abort();
+        }
+    }
+
+    @Override
+    public void run() {
+        try {
+            int attempts = 0;
+            while (!stopped && attempts < maxReconnectAttempts) {
+                attempts++;
+                try {
+                    if (stream()) {
+                        return; // unsupported by server or terminal event seen
+                    }
+                } catch (IOException e) {
+                    if (stopped) {
+                        return;
+                    }
+                    logger.log(Level.FINE, "Build progress stream dropped, reconnecting: " + e.getMessage());
+                }
+                Thread.sleep(RECONNECT_BACKOFF_MS);
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } catch (Exception e) {
+            // progress must never break the build
+            logger.log(Level.FINE, "Build progress consumer stopped: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Opens the SSE stream and forwards events until it ends.
+     * Returns true when the consumer is done for good (server has no
+     * progress support, or a terminal event arrived); false to reconnect.
+     */
+    private boolean stream() throws IOException {
+        HttpGet request = requestFactory.create(jobProgressUrl);
+        request.setHeader("Accept", "text/event-stream");
+        if (lastEventId >= 0) {
+            request.setHeader("Last-Event-ID", Long.toString(lastEventId));
+        }
+        // the stream stays open for the whole build: disable the socket
+        // read timeout for this request; stop() aborts it when the build is done
+        request.setConfig(RequestConfig.custom().setSocketTimeout(0).build());
+        currentRequest = request;
+        try {
+            HttpResponse response = httpClient.execute(request);
+            if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+                // old server or progress disabled; polling still reports completion
+                EntityUtils.consumeQuietly(response.getEntity());
+                return true;
+            }
+            try (BufferedReader reader = new BufferedReader(
+                    new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) {
+                String eventId = null;
+                StringBuilder data = new StringBuilder();
+                String line;
+                while (!stopped && (line = reader.readLine()) != null) {
+                    if (line.isEmpty()) {
+                        // end of one SSE event
+                        if (data.length() > 0 && dispatch(eventId, data.toString())) {
+                            return true; // terminal event
+                        }
+                        eventId = null;
+                        data.setLength(0);
+                    } else if (line.startsWith("id:")) {
+                        eventId = line.substring(3).trim();
+                    } else if (line.startsWith("data:")) {
+                        data.append(line.substring(5).trim());
+                    }
+                    // "event:" names and ":" comments (heartbeats) are ignored
+                }
+            }
+            return stopped;
+        } finally {
+            currentRequest = null;
+        }
+    }
+
+    /** Forwards one event to the listener. Returns true for terminal events. */
+    private boolean dispatch(String eventId, String data) {
+        boolean terminal = false;
+        try {
+            JSONObject json = (JSONObject) new JSONParser().parse(data);
+            if (eventId != null) {
+                lastEventId = Long.parseLong(eventId);
+            }
+            String stage = (String) json.get("stage");
+            String detail = (String) json.get("detail");
+            Number percent = (Number) json.get("percent");
+            Number currentFile = (Number) json.get("currentFile");
+            Number totalFiles = (Number) json.get("totalFiles");
+            Boolean isTerminal = (Boolean) json.get("terminal");
+            terminal = isTerminal != null && isTerminal;
+            listener.onProgress(stage, detail,
+                    percent != null ? percent.intValue() : 0,
+                    currentFile != null ? currentFile.intValue() : -1,
+                    totalFiles != null ? totalFiles.intValue() : -1);
+        } catch (Exception e) {
+            // a malformed event or a listener bug must not kill the stream
+            logger.log(Level.FINE, "Ignoring bad progress event: " + e.getMessage());
+        }
+        return terminal;
+    }
+}