avformat/http: bounded-range continuation never fires on chunked 206 responses, truncating reads at the first request_size boundary

[ronag]

Summary

The fork's bounded-request feature (request_size / initial_request_size, with the tri-state multiple_requests default of -1 and implied keep-alive) relies on http_buf_read() returning AVERROR(EAGAIN) when s->off reaches s->range_end, so that http_read_stream() can issue the next range request on the same connection. That EAGAIN is only produced in the non-chunked read path. If the server answers the bounded range request with 206 Partial Content + Content-Range + Transfer-Encoding: chunked — a combination http_read_header() explicitly supports — the chunk-termination branch runs first, sets s->chunkend = 1, and returns before the continuation check is ever reached. The stream is then treated as a premature EOF and the read fails with AVERROR(EIO) after only the first request_size bytes (or silently truncates with explicit multiple_requests=0). The continuation request for the rest of the file is never sent, even though the keep-alive connection is idle and ready.

Location

• http_buf_read() — early chunkend return and last-chunk handling that short-circuit before the continuation check:

  • FFmpeg/libavformat/http.c

    Lines 1779 to 1781 in 9a83bff

    	if (s->chunkend) {
    	return AVERROR_EOF;
    	}

  • FFmpeg/libavformat/http.c

    Lines 1797 to 1806 in 9a83bff

    	if (!s->chunksize && s->multiple_requests) {
    	http_get_line(s, line, sizeof(line)); // read empty chunk
    	s->chunkend = 1;
    	return 0;
    	}
    	else if (!s->chunksize) {
    	av_log(h, AV_LOG_DEBUG, "Last chunk received, closing conn\n");
    	ffurl_closep(&s->hd);
    	return 0;
    	}

• http_buf_read() — the bounded-range continuation check that becomes unreachable:

  • FFmpeg/libavformat/http.c

    Lines 1823 to 1829 in 9a83bff

    	} else {
    	uint64_t file_end = s->end_off ? s->end_off : s->filesize;
    	uint64_t target_end = s->range_end ? s->range_end : file_end;
    	if ((!s->willclose || s->chunksize == UINT64_MAX) && s->off >= file_end)
    	return AVERROR_EOF;
    	if (s->off == target_end && target_end < file_end)
    	return AVERROR(EAGAIN); /* reached end of content range */

• http_read_stream() — the EAGAIN continuation handler and the premature-EOF → EIO conversion:

  • FFmpeg/libavformat/http.c

    Lines 1908 to 1937 in 9a83bff

    	retry:
    	read_ret = http_buf_read(h, buf, size);
    	while (read_ret < 0) {
    	uint64_t target = h->is_streamed ? 0 : s->off;
    	bool is_premature = s->filesize > 0 && s->off < s->filesize;
    	if (read_ret == AVERROR_EXIT)
    	break;
    	else if (read_ret == AVERROR(EAGAIN)) {
    	/* send new request for more data on existing connection */
    	AVDictionary *options = NULL;
    	if (s->willclose)
    	ffurl_closep(&s->hd);
    	s->initial_requests = 0; /* continue streaming uninterrupted from now on */
    	read_ret = http_open_cnx(h, &options);
    	av_dict_free(&options);
    	if (read_ret == 0)
    	goto retry;
    	}
    	if (h->is_streamed && !s->reconnect_streamed)
    	break;
    	if (!(s->reconnect && is_premature) &&
    	!(s->reconnect_at_eof && read_ret == AVERROR_EOF)) {
    	if (is_premature)
    	return AVERROR(EIO);
    	else
    	break;
    	}

• Implicated fork hunks: multiple_requests default -1 (

  FFmpeg/libavformat/http.c

  Line 199 in 9a83bff

  { "multiple_requests", "use persistent connections", OFFSET(multiple_requests), AV_OPT_TYPE_BOOL, { .i64 = -1 }, -1, 1, D | E },

  ), bounded Range: building (

  FFmpeg/libavformat/http.c

  Lines 1649 to 1666 in 9a83bff

  	int is_partial_request = 0;
  	if (!has_header(s->headers, "\r\nRange: ") && !post && (s->off > 0 || s->end_off || s->seekable != 0)) {
  	av_bprintf(&request, "Range: bytes=%"PRIu64"-", s->off);
  	uint64_t req_size = request_size(h);
  	if (req_size && s->seekable != 0) {
  	uint64_t target_off = s->off + req_size;
  	if (target_off < s->off) /* overflow */
  	target_off = UINT64_MAX;
  	if (s->end_off)
  	target_off = FFMIN(target_off, s->end_off);
  	if (target_off != UINT64_MAX) {
  	av_bprintf(&request, "%"PRId64, target_off - 1);
  	is_partial_request = 1;
  	}
  	} else if (s->end_off)
  	av_bprintf(&request, "%"PRId64, s->end_off - 1);
  	av_bprintf(&request, "\r\n");
  	}

  ), implied keep-alive (

  FFmpeg/libavformat/http.c

  Lines 1670 to 1675 in 9a83bff

  	if (!has_header(s->headers, "\r\nConnection: ")) {
  	int keep_alive = s->multiple_requests > 0;
  	if (s->multiple_requests < 0 /* auto */ && is_partial_request)
  	keep_alive = 1;
  	av_bprintf(&request, "Connection: %s\r\n", keep_alive ? "keep-alive" : "close");
  	}

  )

Details

static int http_buf_read(URLContext *h, uint8_t *buf, int size)
{
    ...
    if (s->chunksize != UINT64_MAX) {
        if (s->chunkend) {
            return AVERROR_EOF;                                  // L1779-1781
        }
        if (!s->chunksize) {
            ...
            s->chunksize = strtoull(line, NULL, 16);
            ...
            if (!s->chunksize && s->multiple_requests) {         // L1797: truthy with default -1
                http_get_line(s, line, sizeof(line)); // read empty chunk
                s->chunkend = 1;
                return 0;                                        // <-- returns here
            }
            else if (!s->chunksize) {
                av_log(h, AV_LOG_DEBUG, "Last chunk received, closing conn\n");
                ffurl_closep(&s->hd);
                return 0;                                        // <-- or here (multiple_requests=0)
            }
            ...
        }
        size = FFMIN(size, s->chunksize);
    }

    /* read bytes from input buffer first */
    len = s->buf_end - s->buf_ptr;
    if (len > 0) {
        ...
    } else {
        uint64_t file_end   = s->end_off   ? s->end_off   : s->filesize;
        uint64_t target_end = s->range_end ? s->range_end : file_end;
        if ((!s->willclose || s->chunksize == UINT64_MAX) && s->off >= file_end)
            return AVERROR_EOF;
        if (s->off == target_end && target_end < file_end)
            return AVERROR(EAGAIN); /* reached end of content range */  // L1828-1829: unreachable for chunked
        ...
    }

Step-by-step trace (e.g. request_size=65536, 1 MB file, server that chunks its 206 responses):

1. Request: with request_size set, http_connect() (L1650-1665) emits Range: bytes=0-65535 and sets is_partial_request = 1; the fork hunk at L1670-1675 then emits Connection: keep-alive because s->multiple_requests defaults to -1 (L199).
2. Response headers 206, Content-Range: bytes 0-65535/1000000, Transfer-Encoding: chunked: process_line() (L1321-1324) sets s->filesize = UINT64_MAX, s->chunksize = 0; parse_content_range() (L957-968) sets s->off = 0, s->range_end = 65536, s->filesize_from_content_range = 1000000; http_read_header() (L1521-1524, "filesize from Content-Range can always be used, even if using chunked Transfer-Encoding") sets s->filesize = 1000000.
3. Body: chunks totaling 65536 bytes are consumed through http_buf_read(); afterwards s->off == 65536, s->chunksize == 0.
4. Next http_buf_read() call: the chunked branch parses the final 0 chunk size, then L1797 if (!s->chunksize && s->multiple_requests) is taken (-1 is truthy): s->chunkend = 1; return 0;. The continuation check at L1828-1829 (s->off == target_end && target_end < file_end → AVERROR(EAGAIN)) is never reached.
5. Subsequent http_buf_read() calls: L1779-1781 return AVERROR_EOF immediately.
6. http_read_stream(): L1912 computes is_premature = s->filesize > 0 && s->off < s->filesize → true (65536 < 1000000). AVERROR_EOF is neither AVERROR_EXIT nor AVERROR(EAGAIN), so the continuation handler at L1916-1926 is skipped; with default reconnect=0 / reconnect_at_eof=0, L1931-1934 hit if (is_premature) return AVERROR(EIO);. The caller gets EIO after 65536 of 1000000 bytes.
7. For comparison, the identical scenario without chunked encoding works: L1828-1829 return AVERROR(EAGAIN) and http_read_stream() L1916-1926 reuses the keep-alive connection via http_open_cnx() to request bytes=65536-.

With explicit multiple_requests=0 the close-branch (L1802-1805) runs instead: it returns 0 with s->hd == NULL, and the next http_read_stream() call returns AVERROR_EOF at L1894-1895 — a silent truncation rather than an error.

Note also that s->chunkend is only ever reset in ff_http_do_new_request() (L598); http_read_header() resets s->chunksize (L1483) but not chunkend, and http_connect()'s input-buffer init (L1719-1728) does not reset it either. Any fix that routes the chunked case through the continuation handler must clear chunkend on the new request, otherwise the stale flag would immediately EOF the follow-up chunked response.

Impact

Severity: medium. Any file fetched with request_size / initial_request_size from a server that applies chunked transfer-encoding to its 206 responses (common behind reverse proxies / CDNs that re-chunk upstream responses) is truncated at the first range boundary:

• With the default multiple_requests=-1: the demuxer receives AVERROR(EIO) mid-file — playback/probing fails outright.
• With explicit multiple_requests=0: a silent, truncated EOF — arguably worse, since the caller believes it read the whole file.

This makes the fork's flagship bounded-request feature silently non-functional against chunked-206 servers, since the keep-alive chunk-end branch is the default code path for that feature.

Suggested fix

In the last-chunk handling, detect that the response was a bounded range covering less than the known file size and return AVERROR(EAGAIN) so the existing continuation logic in http_read_stream() (L1916-1926, which already copes with willclose and a closed s->hd) takes over. Mirror the same check in the chunkend early return so a retried read does not turn into AVERROR_EOF, and reset chunkend in http_connect() so the follow-up response starts clean (it is currently only reset in ff_http_do_new_request()).

--- a/libavformat/http.c
+++ b/libavformat/http.c
@@ static int http_buf_read(URLContext *h, uint8_t *buf, int size)
     if (s->chunksize != UINT64_MAX) {
         if (s->chunkend) {
-            return AVERROR_EOF;
+            uint64_t file_end   = s->end_off   ? s->end_off   : s->filesize;
+            uint64_t target_end = s->range_end ? s->range_end : file_end;
+            if (s->off == target_end && target_end < file_end)
+                return AVERROR(EAGAIN); /* reached end of content range */
+            return AVERROR_EOF;
         }
@@ static int http_buf_read(URLContext *h, uint8_t *buf, int size)
-            if (!s->chunksize && s->multiple_requests) {
-                http_get_line(s, line, sizeof(line)); // read empty chunk
-                s->chunkend = 1;
-                return 0;
-            }
-            else if (!s->chunksize) {
-                av_log(h, AV_LOG_DEBUG, "Last chunk received, closing conn\n");
-                ffurl_closep(&s->hd);
-                return 0;
-            }
+            if (!s->chunksize) {
+                uint64_t file_end   = s->end_off   ? s->end_off   : s->filesize;
+                uint64_t target_end = s->range_end ? s->range_end : file_end;
+                if (s->multiple_requests) {
+                    http_get_line(s, line, sizeof(line)); // read empty chunk
+                    s->chunkend = 1;
+                } else {
+                    av_log(h, AV_LOG_DEBUG, "Last chunk received, closing conn\n");
+                    ffurl_closep(&s->hd);
+                    s->chunkend = 1;
+                }
+                if (s->off == target_end && target_end < file_end)
+                    return AVERROR(EAGAIN); /* request the next range */
+                return 0;
+            }
             else if (s->chunksize == UINT64_MAX) {
@@ static int http_connect(URLContext *h, const char *path, const char *local_path,
     /* init input buffer */
     s->buf_ptr          = s->buffer;
     s->buf_end          = s->buffer;
     s->line_count       = 0;
     s->off              = 0;
     s->icy_data_read    = 0;
     s->filesize         = UINT64_MAX;
     s->willclose        = 0;
+    s->chunkend         = 0;
     s->end_chunked_post = 0;
     s->end_header       = 0;

The s->chunkend = 0; reset in http_connect() is required: without it, the continuation request issued via http_open_cnx() would see the stale chunkend = 1 and return AVERROR_EOF on the first read of the next chunked response (http_read_header() only resets chunksize, not chunkend).

Upstream status

The same short-circuit exists in upstream FFmpeg master (git show upstream/master:libavformat/http.c shows the identical chunkend / last-chunk logic and the same unreachable range-end EAGAIN continuation, plus the matching EAGAIN handler in http_read_stream()). Upstream is however only affected with explicit -multiple_requests 1 against a server that both truncates the requested range and chunks the 206 response, since upstream defaults multiple_requests to 0 and has no request_size / initial_request_size feature. The fork's multiple_requests=-1 default and implied keep-alive make the broken path the default for its bounded-request feature. Worth reporting upstream as well, with the lower-severity framing.