ci: publish otel plugin lambda layer (#619)

* ci: publish otel plugin lambda layer

* ci: make otel plugin layers public

* ci: continue layer publishing across regions

* ci: make layer publishing idempotent

* ci: allow manual layer publishing

* ci: reuse built layer artifacts

* fix: reject layer output inside build directory

* ci: harden layer release workflow

* ci: verify lambda layer artifact hashes

* ci: pin sdk version for otel-only layers

* ci: move layer sdk pin to release metadata

Author: zhongkechen

.github/lambda-layer-publish.toml

@@ -0,0 +1,2 @@
+[layer]
+sdk-version = "1.8.0"

.github/scripts/build_lambda_layer.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import argparse
+import shutil
+import subprocess
+import sys
+import tempfile
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+
+ARCHITECTURE_PLATFORMS = {
+    "x86_64": "manylinux2014_x86_64",
+    "arm64": "manylinux2014_aarch64",
+}
+SUPPORTED_PYTHON_VERSIONS = ("3.11", "3.12", "3.13", "3.14")
+
+
+@dataclass(frozen=True)
+class BuildConfig:
+    output: Path
+    target_python: str
+    architecture: str
+    sdk_distribution: Path
+    otel_distribution: Path
+    build_dir: Path | None = None
+
+
+def build_layer(config: BuildConfig) -> Path:
+    """Build a Lambda layer containing the SDK and OpenTelemetry plugin."""
+
+    _validate_config(config)
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        work_dir = config.build_dir or Path(temp_dir) / "layer"
+        _validate_output_location(config.output, work_dir)
+        if work_dir.exists():
+            shutil.rmtree(work_dir)
+        layer_python_dir = work_dir / "python"
+        layer_python_dir.mkdir(parents=True)
+
+        _install_layer_dependencies(config, layer_python_dir)
+        _write_zip(config.output, work_dir)
+
+    return config.output
+
+
+def _validate_config(config: BuildConfig) -> None:
+    if config.architecture not in ARCHITECTURE_PLATFORMS:
+        supported = ", ".join(sorted(ARCHITECTURE_PLATFORMS))
+        raise ValueError(
+            f"Unsupported architecture: {config.architecture}. "
+            f"Supported architectures: {supported}"
+        )
+
+    if config.target_python not in SUPPORTED_PYTHON_VERSIONS:
+        supported = ", ".join(SUPPORTED_PYTHON_VERSIONS)
+        raise ValueError(
+            f"Unsupported Python version: {config.target_python}. "
+            f"Supported versions: {supported}"
+        )
+
+    for distribution in (config.sdk_distribution, config.otel_distribution):
+        if not distribution.is_file():
+            raise FileNotFoundError(distribution)
+
+
+def _validate_output_location(output: Path, build_dir: Path) -> None:
+    if output.resolve().is_relative_to(build_dir.resolve()):
+        raise ValueError("Layer output must be outside the build directory")
+
+
+def _install_layer_dependencies(config: BuildConfig, target_dir: Path) -> None:
+    python_version = config.target_python
+    abi = f"cp{python_version.replace('.', '')}"
+    platform = ARCHITECTURE_PLATFORMS[config.architecture]
+
+    command = [
+        sys.executable,
+        "-m",
+        "pip",
+        "install",
+        "--upgrade",
+        "--target",
+        str(target_dir),
+        "--platform",
+        platform,
+        "--implementation",
+        "cp",
+        "--python-version",
+        python_version,
+        "--abi",
+        abi,
+        "--only-binary",
+        ":all:",
+        "--no-compile",
+        str(config.sdk_distribution),
+        str(config.otel_distribution),
+    ]
+    subprocess.run(command, check=True)
+
+
+def _write_zip(output: Path, layer_root: Path) -> None:
+    output.parent.mkdir(parents=True, exist_ok=True)
+    if output.exists():
+        output.unlink()
+
+    with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+        for path in sorted(layer_root.rglob("*")):
+            if path.is_file():
+                archive.write(path, path.relative_to(layer_root))
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="build_lambda_layer.py",
+        description="Build the AWS Durable Execution SDK OTel plugin Lambda layer.",
+    )
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument(
+        "--target-python",
+        choices=SUPPORTED_PYTHON_VERSIONS,
+        required=True,
+        help="Lambda Python minor version.",
+    )
+    parser.add_argument(
+        "--architecture",
+        choices=sorted(ARCHITECTURE_PLATFORMS),
+        required=True,
+        help="Lambda instruction set architecture.",
+    )
+    parser.add_argument("--sdk-distribution", type=Path, required=True)
+    parser.add_argument("--otel-distribution", type=Path, required=True)
+    parser.add_argument(
+        "--build-dir",
+        type=Path,
+        help="Optional scratch directory. Existing contents are replaced.",
+    )
+
+    args = parser.parse_args(argv)
+    output = build_layer(
+        BuildConfig(
+            output=args.output,
+            target_python=args.target_python,
+            architecture=args.architecture,
+            sdk_distribution=args.sdk_distribution,
+            otel_distribution=args.otel_distribution,
+            build_dir=args.build_dir,
+        )
+    )
+    print(output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/resolve_layer_sdk_version.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2025-present Amazon.com, Inc. or its affiliates.
+#
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import argparse
+import tomllib
+from pathlib import Path
+
+
+def _read_pinned_sdk_version(metadata_path: Path) -> str:
+    with metadata_path.open("rb") as file:
+        metadata = tomllib.load(file)
+
+    try:
+        pinned_version = metadata["layer"]["sdk-version"]
+    except KeyError as error:
+        raise ValueError(f"{metadata_path}: missing layer.sdk-version") from error
+
+    if not isinstance(pinned_version, str) or not pinned_version:
+        raise ValueError(f"{metadata_path}: layer.sdk-version must be a string")
+    return pinned_version
+
+
+def resolve_layer_sdk_version(
+    event_name: str,
+    release_tag: str,
+    source_sdk_version: str,
+    metadata_path: Path,
+) -> str:
+    """Resolve the SDK version bundled in the OTel plugin Lambda layer."""
+
+    pinned_version = _read_pinned_sdk_version(metadata_path)
+    sdk_release = any(part.startswith("sdk-v") for part in release_tag.split(","))
+
+    if event_name == "release" and not sdk_release:
+        return pinned_version
+
+    if event_name == "release" and pinned_version != source_sdk_version:
+        raise ValueError(
+            f"New SDK releases must update layer.sdk-version to {source_sdk_version}"
+        )
+
+    return source_sdk_version
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Resolve the SDK version for the OTel plugin Lambda layer."
+    )
+    parser.add_argument("--event-name", required=True)
+    parser.add_argument("--release-tag", default="")
+    parser.add_argument("--source-sdk-version", required=True)
+    parser.add_argument("--metadata", type=Path, required=True)
+    args = parser.parse_args(argv)
+
+    print(
+        resolve_layer_sdk_version(
+            event_name=args.event_name,
+            release_tag=args.release_tag,
+            source_sdk_version=args.source_sdk_version,
+            metadata_path=args.metadata,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/tests/test_build_lambda_layer.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+
+from build_lambda_layer import BuildConfig, build_layer
+
+
+def test_build_layer_installs_dependencies_and_zips_lambda_layout(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    sdk_wheel = tmp_path / "aws_durable_execution_sdk_python-1.0.0-py3-none-any.whl"
+    otel_wheel = (
+        tmp_path / "aws_durable_execution_sdk_python_otel-1.0.0-py3-none-any.whl"
+    )
+    sdk_wheel.write_text("sdk")
+    otel_wheel.write_text("otel")
+    commands: list[list[str]] = []
+
+    def fake_run(command: list[str], check: bool) -> subprocess.CompletedProcess[str]:
+        commands.append(command)
+        target = Path(command[command.index("--target") + 1])
+        (target / "aws_durable_execution_sdk_python").mkdir()
+        (target / "aws_durable_execution_sdk_python" / "__init__.py").write_text("")
+        (target / "aws_durable_execution_sdk_python_otel").mkdir()
+        (target / "aws_durable_execution_sdk_python_otel" / "__init__.py").write_text(
+            ""
+        )
+        return subprocess.CompletedProcess(command, 0)
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+
+    output = build_layer(
+        BuildConfig(
+            output=tmp_path / "layer.zip",
+            target_python="3.12",
+            architecture="arm64",
+            sdk_distribution=sdk_wheel,
+            otel_distribution=otel_wheel,
+        )
+    )
+
+    assert output == tmp_path / "layer.zip"
+    assert commands[0][commands[0].index("--platform") + 1] == "manylinux2014_aarch64"
+    assert commands[0][commands[0].index("--abi") + 1] == "cp312"
+    assert "--no-compile" in commands[0]
+    assert str(sdk_wheel) in commands[0]
+    assert str(otel_wheel) in commands[0]
+
+    with zipfile.ZipFile(output) as archive:
+        assert (
+            "python/aws_durable_execution_sdk_python/__init__.py" in archive.namelist()
+        )
+        assert (
+            "python/aws_durable_execution_sdk_python_otel/__init__.py"
+            in archive.namelist()
+        )
+
+
+@pytest.mark.parametrize(
+    ("target_python", "architecture", "error"),
+    [
+        ("3.10", "x86_64", "Unsupported Python version"),
+        ("3.12", "sparc", "Unsupported architecture"),
+    ],
+)
+def test_build_layer_rejects_unsupported_targets(
+    target_python: str,
+    architecture: str,
+    error: str,
+    tmp_path: Path,
+) -> None:
+    sdk_wheel = tmp_path / "sdk.whl"
+    otel_wheel = tmp_path / "otel.whl"
+    sdk_wheel.write_text("sdk")
+    otel_wheel.write_text("otel")
+
+    with pytest.raises(ValueError, match=error):
+        build_layer(
+            BuildConfig(
+                output=tmp_path / "layer.zip",
+                target_python=target_python,
+                architecture=architecture,
+                sdk_distribution=sdk_wheel,
+                otel_distribution=otel_wheel,
+            )
+        )
+
+
+def test_build_layer_requires_built_distributions(tmp_path: Path) -> None:
+    with pytest.raises(FileNotFoundError):
+        build_layer(
+            BuildConfig(
+                output=tmp_path / "layer.zip",
+                target_python="3.13",
+                architecture="x86_64",
+                sdk_distribution=tmp_path / "missing-sdk.whl",
+                otel_distribution=tmp_path / "missing-otel.whl",
+            )
+        )
+
+
+def test_build_layer_rejects_output_inside_build_directory(tmp_path: Path) -> None:
+    sdk_wheel = tmp_path / "sdk.whl"
+    otel_wheel = tmp_path / "otel.whl"
+    sdk_wheel.write_text("sdk")
+    otel_wheel.write_text("otel")
+    build_dir = tmp_path / "layer"
+
+    with pytest.raises(ValueError, match="outside the build directory"):
+        build_layer(
+            BuildConfig(
+                output=build_dir / "layer.zip",
+                target_python="3.13",
+                architecture="x86_64",
+                sdk_distribution=sdk_wheel,
+                otel_distribution=otel_wheel,
+                build_dir=build_dir,
+            )
+        )