From af5b230e78021fe3492803628344313b2de08b32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:00:00 +0000 Subject: [PATCH 1/2] test: add minimal reproduction for inserting binary buffers into String/FixedString (#837) --- .../__tests__/integration/data_types.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index ebd86298b..420fe4740 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -97,6 +97,53 @@ describe("data types", () => { ); }); + // Minimal reproduction for https://github.com/ClickHouse/clickhouse-js/issues/837 + // Inserting raw binary buffer data into a String/FixedString field via + // JSONEachRow does not work: the JSON value is interpreted as UTF-8 text, so + // any byte >= 0x80 is re-encoded into a multi-byte UTF-8 sequence. For a + // FixedString this overflows the declared size (TOO_LARGE_STRING_SIZE), and + // even for a plain String the stored bytes no longer match the original ones. + it("cannot insert raw binary buffer data into FixedString via JSONEachRow (#837)", async () => { + const table = await createTableWithFields(client, "fs FixedString(2)"); + // Two bytes that fit into FixedString(2) as raw bytes, but become four + // bytes once encoded as UTF-8 text (each 0xF0 turns into 0xEF 0xBF 0xBD). + const binary = Buffer.from([0xf0, 0xf0]); + await expect( + client.insert({ + table, + values: [{ fs: binary.toString("binary") }], + format: "JSONEachRow", + }), + ).rejects.toMatchObject( + expect.objectContaining({ + type: "TOO_LARGE_STRING_SIZE", + message: expect.stringContaining("Too large value for FixedString(2)"), + }), + ); + }); + + it("silently corrupts binary buffer data inserted into String via JSONEachRow (#837)", async () => { + const table = await createTableWithFields(client, "s String"); + const binary = Buffer.from([0xf0, 0xf0]); + await client.insert({ + table, + values: [{ id: 1, s: binary.toString("binary") }], + format: "JSONEachRow", + }); + const [{ stored }] = await client + .query({ + query: `SELECT hex(s) AS stored FROM ${table}`, + format: "JSONEachRow", + }) + .then((r) => r.json<{ stored: string }>()); + // The original bytes were F0F0, but `toString("binary")` maps each byte to + // a Latin-1 code point that is then re-encoded as UTF-8 on the way in + // (0xF0 -> U+00F0 -> 0xC3 0xB0), so the round-tripped value no longer + // matches the input buffer. + expect(stored).not.toBe("F0F0"); + expect(stored).toBe("C3B0C3B0"); + }); + it("should work with decimals", async () => { const row1 = { id: 1, From 0ad8b6c84cb9c97d752ca1aba8da9a18aecb4c9c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 18 Jun 2026 20:09:12 +0000 Subject: [PATCH 2/2] test: move #837 binary buffer reproduction from client-common to client-node --- .../__tests__/integration/data_types.test.ts | 48 --------------- .../node_insert_binary_string.test.ts | 61 +++++++++++++++++++ 2 files changed, 61 insertions(+), 48 deletions(-) create mode 100644 packages/client-node/__tests__/integration/node_insert_binary_string.test.ts diff --git a/packages/client-common/__tests__/integration/data_types.test.ts b/packages/client-common/__tests__/integration/data_types.test.ts index 420fe4740..b5bcd0f60 100644 --- a/packages/client-common/__tests__/integration/data_types.test.ts +++ b/packages/client-common/__tests__/integration/data_types.test.ts @@ -96,54 +96,6 @@ describe("data types", () => { }), ); }); - - // Minimal reproduction for https://github.com/ClickHouse/clickhouse-js/issues/837 - // Inserting raw binary buffer data into a String/FixedString field via - // JSONEachRow does not work: the JSON value is interpreted as UTF-8 text, so - // any byte >= 0x80 is re-encoded into a multi-byte UTF-8 sequence. For a - // FixedString this overflows the declared size (TOO_LARGE_STRING_SIZE), and - // even for a plain String the stored bytes no longer match the original ones. - it("cannot insert raw binary buffer data into FixedString via JSONEachRow (#837)", async () => { - const table = await createTableWithFields(client, "fs FixedString(2)"); - // Two bytes that fit into FixedString(2) as raw bytes, but become four - // bytes once encoded as UTF-8 text (each 0xF0 turns into 0xEF 0xBF 0xBD). - const binary = Buffer.from([0xf0, 0xf0]); - await expect( - client.insert({ - table, - values: [{ fs: binary.toString("binary") }], - format: "JSONEachRow", - }), - ).rejects.toMatchObject( - expect.objectContaining({ - type: "TOO_LARGE_STRING_SIZE", - message: expect.stringContaining("Too large value for FixedString(2)"), - }), - ); - }); - - it("silently corrupts binary buffer data inserted into String via JSONEachRow (#837)", async () => { - const table = await createTableWithFields(client, "s String"); - const binary = Buffer.from([0xf0, 0xf0]); - await client.insert({ - table, - values: [{ id: 1, s: binary.toString("binary") }], - format: "JSONEachRow", - }); - const [{ stored }] = await client - .query({ - query: `SELECT hex(s) AS stored FROM ${table}`, - format: "JSONEachRow", - }) - .then((r) => r.json<{ stored: string }>()); - // The original bytes were F0F0, but `toString("binary")` maps each byte to - // a Latin-1 code point that is then re-encoded as UTF-8 on the way in - // (0xF0 -> U+00F0 -> 0xC3 0xB0), so the round-tripped value no longer - // matches the input buffer. - expect(stored).not.toBe("F0F0"); - expect(stored).toBe("C3B0C3B0"); - }); - it("should work with decimals", async () => { const row1 = { id: 1, diff --git a/packages/client-node/__tests__/integration/node_insert_binary_string.test.ts b/packages/client-node/__tests__/integration/node_insert_binary_string.test.ts new file mode 100644 index 000000000..9d672fd64 --- /dev/null +++ b/packages/client-node/__tests__/integration/node_insert_binary_string.test.ts @@ -0,0 +1,61 @@ +import type { ClickHouseClient } from "@clickhouse/client-common"; +import { describe, it, beforeEach, afterEach, expect } from "vitest"; +import { createTableWithFields } from "@test/fixtures/table_with_fields"; +import { createTestClient } from "@test/utils/client"; + +// Minimal reproduction for https://github.com/ClickHouse/clickhouse-js/issues/837 +// Inserting raw binary buffer data into a String/FixedString field via +// JSONEachRow does not work: the JSON value is interpreted as UTF-8 text, so +// any byte >= 0x80 is re-encoded into a multi-byte UTF-8 sequence. For a +// FixedString this overflows the declared size (TOO_LARGE_STRING_SIZE), and +// even for a plain String the stored bytes no longer match the original ones. +describe("[Node.js] insert binary buffer into String/FixedString (#837)", () => { + let client: ClickHouseClient; + beforeEach(() => { + client = createTestClient(); + }); + afterEach(async () => { + await client.close(); + }); + + it("cannot insert raw binary buffer data into FixedString via JSONEachRow", async () => { + const table = await createTableWithFields(client, "fs FixedString(2)"); + // Two bytes that fit into FixedString(2) as raw bytes, but each 0xF0 byte + // becomes 0xC3 0xB0 once re-encoded as UTF-8 text, overflowing the field. + const binary = Buffer.from([0xf0, 0xf0]); + await expect( + client.insert({ + table, + values: [{ fs: binary.toString("binary") }], + format: "JSONEachRow", + }), + ).rejects.toMatchObject( + expect.objectContaining({ + type: "TOO_LARGE_STRING_SIZE", + message: expect.stringContaining("Too large value for FixedString(2)"), + }), + ); + }); + + it("silently corrupts binary buffer data inserted into String via JSONEachRow", async () => { + const table = await createTableWithFields(client, "s String"); + const binary = Buffer.from([0xf0, 0xf0]); + await client.insert({ + table, + values: [{ id: 1, s: binary.toString("binary") }], + format: "JSONEachRow", + }); + const [{ stored }] = await client + .query({ + query: `SELECT hex(s) AS stored FROM ${table}`, + format: "JSONEachRow", + }) + .then((r) => r.json<{ stored: string }>()); + // The original bytes were F0F0, but `toString("binary")` maps each byte to + // a Latin-1 code point that is then re-encoded as UTF-8 on the way in + // (0xF0 -> U+00F0 -> 0xC3 0xB0), so the round-tripped value no longer + // matches the input buffer. + expect(stored).not.toBe("F0F0"); + expect(stored).toBe("C3B0C3B0"); + }); +});