ARSN-625: add per part checksums to object metadata

lib/models/ObjectMD.ts

@@ -5,7 +5,8 @@ import ObjectMDLocation, { ObjectMDLocationData, Location } from './ObjectMDLoca
 import ObjectMDAmzRestore from './ObjectMDAmzRestore';
 import ObjectMDArchive from './ObjectMDArchive';
 import { ObjectMDAzureInfoMetadata } from './ObjectMDAzureInfo';
-import ObjectMDChecksum, { ChecksumAlgorithm, ChecksumType } from './ObjectMDChecksum';
+import ObjectMDChecksum, { ChecksumAlgorithm, ChecksumType, partCountFromSuffix } from './ObjectMDChecksum';
+import { PartChecksum, isValidPartChecksums } from './ObjectMDPartChecksums';
 
 export type ACL = {
     Canned: string;
@@ -122,6 +123,10 @@ export type ObjectMDData = {
     // This is only set when it differs from `owner-id`.
     bucketOwnerId?: string;
     checksum?: ObjectMDChecksum;
+    // Checksum of each part of a completed COMPOSITE MPU, one entry per part.
+    // Part checksums must survive any location change so they are stored at the top level of the object
+    // and not in the location array.
+    partChecksums?: PartChecksum[];
 };
 
 /**
@@ -538,6 +543,53 @@ export default class ObjectMD {
         return this._data.checksum ?? null;
     }
 
+    /**
+     * Set the checksum of each part of a completed COMPOSITE MPU.
+     *
+     * Requires the COMPOSITE object-level checksum and the content length to be
+     * set first: the digests are validated against the checksum algorithm, and
+     * the part sizes must add up to the content length.
+     *
+     * @param parts - one entry per part, in part-number order
+     * @return itself
+     */
+    setPartChecksums(parts: PartChecksum[]) {
+        const checksum = this._data.checksum;
+        if (!checksum) {
+            throw new Error('partChecksums require an object-level checksum to be set first.');
+        }
+        if (checksum.checksumType !== 'COMPOSITE') {
+            throw new Error(`partChecksums are only valid for COMPOSITE checksums, not ${checksum.checksumType}.`);
+        }
+        const error = isValidPartChecksums(parts, checksum.checksumAlgorithm, this._data['content-length']);
+        if (error !== null) {
+            throw new Error(error);
+        }
+        const etagParts = partCountFromSuffix(this._data['content-md5']);
+        if (etagParts !== null && etagParts !== parts.length) {
+            throw new Error(
+                `got ${parts.length} part checksums, but the ETag ends in "-${etagParts}", ` +
+                    `meaning ${etagParts} parts: ${this._data['content-md5']}`,
+            );
+        }
+        const checksumParts = partCountFromSuffix(checksum.checksumValue);
+        if (checksumParts !== null && checksumParts !== parts.length) {
+            throw new Error(
+                `got ${parts.length} part checksums, but the object checksum ends in ` +
+                    `"-${checksumParts}", meaning ${checksumParts} parts: ${checksum.checksumValue}`,
+            );
+        }
+        this._data.partChecksums = parts;
+        return this;
+    }
+
+    /**
+     * Returns the checksum of each part, or null if not set.
+     */
+    getPartChecksums(): PartChecksum[] | null {
+        return this._data.partChecksums ?? null;
+    }
+
     /**
      * Set content-language
      *

lib/models/ObjectMDChecksum.ts

@@ -1,8 +1,8 @@
 export const CHECKSUM_ALGORITHMS = ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256'] as const;
 export const CHECKSUM_TYPES = ['FULL_OBJECT', 'COMPOSITE'] as const;
 
-export type ChecksumAlgorithm = typeof CHECKSUM_ALGORITHMS[number];
-export type ChecksumType = typeof CHECKSUM_TYPES[number];
+export type ChecksumAlgorithm = (typeof CHECKSUM_ALGORITHMS)[number];
+export type ChecksumType = (typeof CHECKSUM_TYPES)[number];
 
 export const CHECKSUM_XML_TAGS: Record<ChecksumAlgorithm, string> = {
     crc32: 'ChecksumCRC32',
@@ -15,23 +15,32 @@ export const CHECKSUM_XML_TAGS: Record<ChecksumAlgorithm, string> = {
 const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
 
 const digestLengths: Record<ChecksumAlgorithm, number> = {
-    crc32:     8,
-    crc32c:    8,
+    crc32: 8,
+    crc32c: 8,
     crc64nvme: 12,
-    sha1:      28,
-    sha256:    44,
+    sha1: 28,
+    sha256: 44,
 };
 
-function isValidDigestValue(algorithm: ChecksumAlgorithm, value: string): boolean {
+/**
+ * Validate a raw base64 digest for an algorithm.
+ */
+export function isValidDigestValue(algorithm: ChecksumAlgorithm, value: string): boolean {
     const digestLength = digestLengths[algorithm];
     return typeof value === 'string' && value.length === digestLength && base64Regex.test(value);
 }
 
-function isValidDigest(
-    algorithm: ChecksumAlgorithm,
-    value: string,
-    checksumType: ChecksumType,
-): boolean {
+/**
+ * Extract the part count from a value ending in "-<partCount>". A multipart
+ * object carries that suffix on both its ETag and, for a COMPOSITE MPU, on its checksum
+ * value. Returns null when there is no such suffix.
+ */
+export function partCountFromSuffix(value: string): number | null {
+    const match = /-([1-9][0-9]*)$/.exec(value);
+    return match === null ? null : Number.parseInt(match[1], 10);
+}
+
+function isValidDigest(algorithm: ChecksumAlgorithm, value: string, checksumType: ChecksumType): boolean {
     if (checksumType === 'COMPOSITE') {
         if (isValidDigestValue(algorithm, value)) {
             return true;
@@ -77,11 +86,7 @@ export default class ObjectMDChecksum {
         return null;
     }
 
-    constructor(
-        checksumAlgorithm: ChecksumAlgorithm,
-        checksumValue: string,
-        checksumType: ChecksumType,
-    ) {
+    constructor(checksumAlgorithm: ChecksumAlgorithm, checksumValue: string, checksumType: ChecksumType) {
         const error = ObjectMDChecksum.isValid({ checksumAlgorithm, checksumValue, checksumType });
         if (error !== null) {
             throw new Error(error);

lib/models/ObjectMDPartChecksums.ts

@@ -0,0 +1,60 @@
+import { ChecksumAlgorithm, isValidDigestValue } from './ObjectMDChecksum';
+
+export const MAX_PART_NUMBER = 10000;
+
+export type PartChecksum = {
+    /** Part number: starts at 1 and must be contiguous (1..N). */
+    partNumber: number;
+    /** Byte length of the part. */
+    size: number;
+    /** Raw base64 digest of the part. */
+    checksumValue: string;
+};
+
+/**
+ * Validate the per-part checksums of a completed COMPOSITE MPU: one entry per
+ * part of the object, in part-number order.
+ *
+ * @param parts - the per-part checksums to validate
+ * @param algorithm - the MPU checksum algorithm
+ * @param contentLength - size of the whole object, which the part sizes must
+ *   add up to
+ * @return null if valid, else a description of the first problem found
+ */
+export function isValidPartChecksums(
+    parts: unknown,
+    algorithm: ChecksumAlgorithm,
+    contentLength: number,
+): string | null {
+    if (!Array.isArray(parts)) {
+        return 'partChecksums must be an array';
+    }
+    if (parts.length === 0) {
+        return 'partChecksums must not be empty';
+    }
+    if (parts.length > MAX_PART_NUMBER) {
+        return `partChecksums must hold at most ${MAX_PART_NUMBER} parts, got ${parts.length}`;
+    }
+    let totalSize = 0;
+    for (let i = 0; i < parts.length; i++) {
+        const part = parts[i];
+        if (part === null || typeof part !== 'object') {
+            return `invalid part checksum at index ${i}: not an object`;
+        }
+        const { partNumber, size, checksumValue } = part as PartChecksum;
+        if (partNumber !== i + 1) {
+            return `invalid partNumber at index ${i}: expected ${i + 1}, got ${partNumber}`;
+        }
+        if (!Number.isInteger(size) || size < 0) {
+            return `invalid size for part ${partNumber}: ${size}`;
+        }
+        if (!isValidDigestValue(algorithm, checksumValue)) {
+            return `invalid checksumValue for part ${partNumber}: ${checksumValue}`;
+        }
+        totalSize += size;
+    }
+    if (totalSize !== contentLength) {
+        return `part sizes add up to ${totalSize}, but the object is ${contentLength} bytes`;
+    }
+    return null;
+}

lib/models/index.ts

@@ -12,6 +12,8 @@ export type { BackendKey } from './ObjectMD';
 export { default as ObjectMDAmzRestore } from './ObjectMDAmzRestore';
 export { default as ObjectMDChecksum, CHECKSUM_ALGORITHMS, CHECKSUM_TYPES } from './ObjectMDChecksum';
 export type { ChecksumAlgorithm, ChecksumType } from './ObjectMDChecksum';
+export { isValidPartChecksums, MAX_PART_NUMBER } from './ObjectMDPartChecksums';
+export type { PartChecksum } from './ObjectMDPartChecksums';
 export { default as ObjectMDArchive } from './ObjectMDArchive';
 export { default as ObjectMDAzureInfo } from './ObjectMDAzureInfo';
 export { default as ObjectMDLocation } from './ObjectMDLocation';

package.json

@@ -3,7 +3,7 @@
     "engines": {
         "node": ">=20"
     },
-    "version": "8.5.14",
+    "version": "8.5.15",
     "config": {
         "mongodbMemoryServer": {
             "version": "8.0.23"

tests/unit/models/ObjectMD.spec.js

@@ -1038,3 +1038,109 @@ describe('ObjectMD checksum', () => {
         assert.strictEqual(c.checksumType, 'COMPOSITE');
     });
 });
+
+describe('ObjectMD partChecksums', () => {
+    const part1 = '3AyUPBM2N2sY3QjN2J9wYUzQpOWWybOVJUp6LuJEMf0=';
+    const part2 = 'sgad/t6kdx5RjCEaYWRKlxFEJBzQE6Zz+cDhA72e3m4=';
+    const part3 = '6xvj56XlEPtCZATsAFHleo2Rw1/rC88oQoRzQTCbb24=';
+    const objectDigest = 'Tn0yg0fVZ7+nKLrkZLhCxRMKrI2y0P+6VlGTnC3gic8=';
+    const partSize = 5242880;
+    const parts = [
+        { partNumber: 1, size: partSize, checksumValue: part1 },
+        { partNumber: 2, size: partSize, checksumValue: part2 },
+        { partNumber: 3, size: partSize, checksumValue: part3 },
+    ];
+
+    // An ObjectMD in the state CompleteMPU leaves it in for a 3-part
+    // SHA256 COMPOSITE upload, minus the part checksums themselves.
+    function compositeMd() {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 3);
+        md.setChecksum(new ObjectMDChecksum('sha256', `${objectDigest}-3`, 'COMPOSITE'));
+        return md;
+    }
+
+    it('should return null when no part checksums are set', () => {
+        assert.strictEqual(new ObjectMD().getPartChecksums(), null);
+    });
+
+    it('should store and return the part checksums', () => {
+        const md = compositeMd().setPartChecksums(parts);
+        assert.deepStrictEqual(md.getPartChecksums(), parts);
+    });
+
+    it('should preserve the part checksums through a JSON round-trip', () => {
+        const md = compositeMd().setPartChecksums(parts);
+        const { result } = ObjectMD.createFromBlob(md.getSerialized());
+        assert(result !== undefined);
+        assert.deepStrictEqual(result.getPartChecksums(), parts);
+    });
+
+    it('should not appear in the model attributes when unset', () => {
+        assert.strictEqual(ObjectMD.getAttributes().partChecksums, undefined);
+    });
+
+    it('should throw when no object-level checksum is set', () => {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 3);
+        assert.throws(() => md.setPartChecksums(parts), /require an object-level checksum/);
+    });
+
+    it('should throw when the object-level checksum is not COMPOSITE', () => {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 3);
+        md.setChecksum(new ObjectMDChecksum('sha256', objectDigest, 'FULL_OBJECT'));
+        assert.throws(() => md.setPartChecksums(parts), /only valid for COMPOSITE checksums, not FULL_OBJECT/);
+    });
+
+    it('should throw when the part checksums are malformed', () => {
+        const md = compositeMd();
+        const gapped = [parts[0], { partNumber: 3, size: partSize, checksumValue: part3 }];
+        assert.throws(() => md.setPartChecksums(gapped), /expected 2, got 3/);
+    });
+
+    it('should throw when the sizes do not add up to the content length', () => {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 2);
+        md.setChecksum(new ObjectMDChecksum('sha256', `${objectDigest}-3`, 'COMPOSITE'));
+        assert.throws(() => md.setPartChecksums(parts), /part sizes add up to/);
+    });
+
+    it('should throw when the content length has not been set yet', () => {
+        const md = new ObjectMD();
+        md.setChecksum(new ObjectMDChecksum('sha256', `${objectDigest}-3`, 'COMPOSITE'));
+        assert.throws(() => md.setPartChecksums(parts), /but the object is 0 bytes/);
+    });
+
+    it('should throw when the part count disagrees with the ETag suffix', () => {
+        const md = compositeMd();
+        md.setContentMd5('c763e901f8746cfdc1f21396ca9ce977-4');
+        assert.throws(() => md.setPartChecksums(parts), /the ETag ends in "-4", meaning 4 parts/);
+    });
+
+    it('should accept a matching ETag suffix', () => {
+        const md = compositeMd();
+        md.setContentMd5('c763e901f8746cfdc1f21396ca9ce977-3');
+        assert.deepStrictEqual(md.setPartChecksums(parts).getPartChecksums(), parts);
+    });
+
+    it('should skip the ETag check when the ETag carries no suffix', () => {
+        const md = compositeMd();
+        md.setContentMd5('c763e901f8746cfdc1f21396ca9ce977');
+        assert.deepStrictEqual(md.setPartChecksums(parts).getPartChecksums(), parts);
+    });
+
+    it('should throw when the part count disagrees with the checksum suffix', () => {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 3);
+        md.setChecksum(new ObjectMDChecksum('sha256', `${objectDigest}-4`, 'COMPOSITE'));
+        assert.throws(() => md.setPartChecksums(parts), /the object checksum ends in "-4", meaning 4 parts/);
+    });
+
+    it('should skip the checksum check when the checksum carries no suffix', () => {
+        const md = new ObjectMD();
+        md.setContentLength(partSize * 3);
+        md.setChecksum(new ObjectMDChecksum('sha256', objectDigest, 'COMPOSITE'));
+        assert.deepStrictEqual(md.setPartChecksums(parts).getPartChecksums(), parts);
+    });
+});