From aa7ee44ede6c523b50dfe8ec31f41fdb21881a9a Mon Sep 17 00:00:00 2001 From: Swaraj Mali Date: Sat, 8 Aug 2026 13:48:09 +0530 Subject: [PATCH 1/5] feat: Add interview-scorecard-calibrator kit --- .../.env.example | 7 + .../interview-scorecard-calibrator/.gitignore | 5 + kits/interview-scorecard-calibrator/README.md | 110 + kits/interview-scorecard-calibrator/agent.md | 39 + .../apps/.env.example | 7 + .../apps/.gitignore | 23 + .../apps/README.md | 11 + .../apps/actions/orchestrate.ts | 109 + .../apps/app/globals.css | 33 + .../apps/app/layout.tsx | 35 + .../apps/app/page.tsx | 359 + .../apps/lib/lamatic-client.ts | 34 + .../apps/lib/scorecard.ts | 130 + .../apps/next.config.ts | 5 + .../apps/orchestrate.js | 29 + .../apps/package-lock.json | 7960 +++++++++++++++++ .../apps/package.json | 37 + .../apps/postcss.config.mjs | 7 + .../apps/tsconfig.json | 30 + .../constitutions/default.md | 27 + .../flows/calibrate-scorecard.ts | 302 + .../lamatic.config.ts | 26 + .../calibrate-scorecard_calibrate.ts | 19 + .../calibrate-scorecard_compose.ts | 20 + .../calibrate-scorecard_calibrate_system.md | 39 + .../calibrate-scorecard_calibrate_user.md | 52 + .../calibrate-scorecard_compose_system.md | 16 + .../calibrate-scorecard_compose_user.md | 21 + .../scripts/calibrate-scorecard_normalize.ts | 61 + 29 files changed, 9553 insertions(+) create mode 100644 kits/interview-scorecard-calibrator/.env.example create mode 100644 kits/interview-scorecard-calibrator/.gitignore create mode 100644 kits/interview-scorecard-calibrator/README.md create mode 100644 kits/interview-scorecard-calibrator/agent.md create mode 100644 kits/interview-scorecard-calibrator/apps/.env.example create mode 100644 kits/interview-scorecard-calibrator/apps/.gitignore create mode 100644 kits/interview-scorecard-calibrator/apps/README.md create mode 100644 kits/interview-scorecard-calibrator/apps/actions/orchestrate.ts create mode 100644 kits/interview-scorecard-calibrator/apps/app/globals.css create mode 100644 kits/interview-scorecard-calibrator/apps/app/layout.tsx create mode 100644 kits/interview-scorecard-calibrator/apps/app/page.tsx create mode 100644 kits/interview-scorecard-calibrator/apps/lib/lamatic-client.ts create mode 100644 kits/interview-scorecard-calibrator/apps/lib/scorecard.ts create mode 100644 kits/interview-scorecard-calibrator/apps/next.config.ts create mode 100644 kits/interview-scorecard-calibrator/apps/orchestrate.js create mode 100644 kits/interview-scorecard-calibrator/apps/package-lock.json create mode 100644 kits/interview-scorecard-calibrator/apps/package.json create mode 100644 kits/interview-scorecard-calibrator/apps/postcss.config.mjs create mode 100644 kits/interview-scorecard-calibrator/apps/tsconfig.json create mode 100644 kits/interview-scorecard-calibrator/constitutions/default.md create mode 100644 kits/interview-scorecard-calibrator/flows/calibrate-scorecard.ts create mode 100644 kits/interview-scorecard-calibrator/lamatic.config.ts create mode 100644 kits/interview-scorecard-calibrator/model-configs/calibrate-scorecard_calibrate.ts create mode 100644 kits/interview-scorecard-calibrator/model-configs/calibrate-scorecard_compose.ts create mode 100644 kits/interview-scorecard-calibrator/prompts/calibrate-scorecard_calibrate_system.md create mode 100644 kits/interview-scorecard-calibrator/prompts/calibrate-scorecard_calibrate_user.md create mode 100644 kits/interview-scorecard-calibrator/prompts/calibrate-scorecard_compose_system.md create mode 100644 kits/interview-scorecard-calibrator/prompts/calibrate-scorecard_compose_user.md create mode 100644 kits/interview-scorecard-calibrator/scripts/calibrate-scorecard_normalize.ts diff --git a/kits/interview-scorecard-calibrator/.env.example b/kits/interview-scorecard-calibrator/.env.example new file mode 100644 index 000000000..5a6a5ea02 --- /dev/null +++ b/kits/interview-scorecard-calibrator/.env.example @@ -0,0 +1,7 @@ +# Flow Configurations +CALIBRATE_SCORECARD_FLOW_ID=your-lamatic-flow-uuid-here + +# API Credentials +LAMATIC_API_KEY=your-lamatic-client-api-key-here +LAMATIC_PROJECT_ID=your-lamatic-project-id-here +LAMATIC_API_URL=https://your-organization-project-dev.lamatic.dev diff --git a/kits/interview-scorecard-calibrator/.gitignore b/kits/interview-scorecard-calibrator/.gitignore new file mode 100644 index 000000000..e916ce5aa --- /dev/null +++ b/kits/interview-scorecard-calibrator/.gitignore @@ -0,0 +1,5 @@ +.lamatic/ +node_modules/ +.next/ +.env +.env.local diff --git a/kits/interview-scorecard-calibrator/README.md b/kits/interview-scorecard-calibrator/README.md new file mode 100644 index 000000000..a0803510e --- /dev/null +++ b/kits/interview-scorecard-calibrator/README.md @@ -0,0 +1,110 @@ +# Interview Scorecard Calibrator + +Reconcile multi-interviewer panel feedback into a calibrated hiring scorecard. + +## Problem + +After a panel interview, hiring managers receive fragmented notes — different formats, uneven scoring rigor, and conflicting opinions. Reconciling them into a fair hire/no-hire decision is slow and inconsistent. + +## Solution + +Paste a role rubric and interviewer notes. The Lamatic flow returns: + +- Per-competency calibrated scores with evidence +- Interviewer disagreement map +- Hire / lean-hire / lean-no / no-hire recommendation + confidence +- Follow-up questions for missing evidence +- Internal decision-summary email draft +- A markdown hiring-committee brief + +## Why this kit is unique + +Unlike resume screeners (`hiring`, `hiring-copilot-agent`, `candidate-screener`) or candidate self-reflection coaches (`interview-reflection-coach`), this kit focuses on **post-panel calibration** for hiring managers. + +## Providers & Prerequisites + +- Lamatic account + deployed flow (import or rebuild from this kit) +- LLM provider configured in Lamatic Studio (Gemini / OpenAI / etc.) +- Node.js 18+ + +## Setup in Lamatic Studio + +1. Create a project in [studio.lamatic.ai](https://studio.lamatic.ai) +2. Recreate the flow from `flows/calibrate-scorecard.ts` (or import if your Studio supports kit import) +3. Attach prompts from `prompts/`, model configs from `model-configs/`, and the normalize script from `scripts/` +4. Bind a text generation model on both LLM nodes +5. Deploy the flow and copy the **Flow ID** + +## Run locally + +```bash +cd kits/interview-scorecard-calibrator/apps +cp .env.example .env.local +# fill in LAMATIC_API_URL, LAMATIC_PROJECT_ID, LAMATIC_API_KEY, CALIBRATE_SCORECARD_FLOW_ID +npm install +npm run dev +``` + +Open [http://localhost:3000](http://localhost:3000). + +## Environment variables + +Copy either kit-root `.env.example` or `apps/.env.example` into `apps/.env.local`. + +| Variable | Description | +|---|---| +| `LAMATIC_API_URL` | Lamatic API endpoint | +| `LAMATIC_PROJECT_ID` | Project ID | +| `LAMATIC_API_KEY` | API key | +| `CALIBRATE_SCORECARD_FLOW_ID` | Deployed flow ID for `calibrate-scorecard` | + +## Interviewer notes format + +Provide **at least two** interviewer entries, separated by: + +- a line containing only `---`, and/or +- headings like `Interviewer 1:` / `Interviewer 2 (Bob, Eng Manager):` + +Requests with fewer than two interviewer blocks are rejected before the model runs. + +## Sample input + +**Job title:** Senior Backend Engineer +**Level:** L5 + +**Rubric:** +``` +System Design (High): scalable services, tradeoffs, reliability +Coding (High): correctness, clarity, edge cases +Ownership (Medium): end-to-end delivery, communication +``` + +**Interviewer notes:** +``` +Interviewer 1 (Alice, Staff Eng): +Strong system design around caching and failover. Coding solid but a bit slow. Ownership examples were concrete. Score: Design 4, Coding 3, Ownership 4. + +--- + +Interviewer 2 (Bob, Eng Manager): +Design felt hand-wavy on consistency. Coding was clean. Concerned about stakeholder communication. Score: Design 2, Coding 4, Ownership 2. +``` + +## Deploy + +Use the Vercel deploy link in `lamatic.config.ts` (`links.deploy`) with `root-directory=kits/interview-scorecard-calibrator/apps`. + +## Project structure + +``` +kits/interview-scorecard-calibrator/ +├── lamatic.config.ts +├── agent.md +├── README.md +├── constitutions/default.md +├── flows/calibrate-scorecard.ts +├── prompts/ +├── model-configs/ +├── scripts/ +└── apps/ # Next.js UI +``` diff --git a/kits/interview-scorecard-calibrator/agent.md b/kits/interview-scorecard-calibrator/agent.md new file mode 100644 index 000000000..a229a4273 --- /dev/null +++ b/kits/interview-scorecard-calibrator/agent.md @@ -0,0 +1,39 @@ +# Interview Scorecard Calibrator + +## Overview +This kit helps hiring managers reconcile fragmented multi-interviewer feedback into a calibrated competency scorecard, disagreement map, hire/no-hire recommendation, and an internal decision-summary email draft. It is a single Lamatic flow invoked by the included Next.js app. + +## Purpose +After panel interviews, notes arrive in different formats with uneven scoring rigor. Manual reconciliation is slow and inconsistent. This agent turns a shared rubric plus interviewer notes into a structured decision aid so committees can focus on evidence and disagreements instead of formatting. + +## Flows + +### `calibrate-scorecard` + +- Trigger: on-demand API execution from the kit UI or any Lamatic SDK caller +- Inputs: + - `job_title` (required) + - `level` (optional) + - `rubric` (required) — competency list with weights/expectations + - `interviewer_notes` (required) — notes from 2–6 interviewers +- Pipeline: + 1. **Normalize Inputs** (`codeNode`) — validates and packages context + 2. **Calibrate Scorecard** (`InstructorLLMNode`) — structured JSON scorecard + 3. **Compose Brief** (`LLMNode`) — markdown hiring-committee brief + 4. **API Response** — returns `scorecard` + `brief` +- Outputs: + - `scorecard` — competencies, disagreements, recommendation, confidence, follow-ups, email draft + - `brief` — readable markdown summary for the hiring committee + +## Guardrails +- Decision aid only — humans make the final hiring call +- Do not fabricate interview evidence +- Avoid protected-class inferences +- Keep candidate/interviewer content confidential +- See `constitutions/default.md` + +## Environment +- `LAMATIC_API_URL` +- `LAMATIC_PROJECT_ID` +- `LAMATIC_API_KEY` +- `CALIBRATE_SCORECARD_FLOW_ID` diff --git a/kits/interview-scorecard-calibrator/apps/.env.example b/kits/interview-scorecard-calibrator/apps/.env.example new file mode 100644 index 000000000..5a6a5ea02 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/.env.example @@ -0,0 +1,7 @@ +# Flow Configurations +CALIBRATE_SCORECARD_FLOW_ID=your-lamatic-flow-uuid-here + +# API Credentials +LAMATIC_API_KEY=your-lamatic-client-api-key-here +LAMATIC_PROJECT_ID=your-lamatic-project-id-here +LAMATIC_API_URL=https://your-organization-project-dev.lamatic.dev diff --git a/kits/interview-scorecard-calibrator/apps/.gitignore b/kits/interview-scorecard-calibrator/apps/.gitignore new file mode 100644 index 000000000..053d82325 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/.gitignore @@ -0,0 +1,23 @@ +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions +/coverage +/.next/ +/out/ +/build +.DS_Store +*.pem +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* +.env* +!.env.example +.vercel +*.tsbuildinfo +next-env.d.ts diff --git a/kits/interview-scorecard-calibrator/apps/README.md b/kits/interview-scorecard-calibrator/apps/README.md new file mode 100644 index 000000000..57ab01686 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/README.md @@ -0,0 +1,11 @@ +# Interview Scorecard Calibrator App + +Next.js UI for the Interview Scorecard Calibrator kit. + +```bash +cp .env.example .env.local +npm install +npm run dev +``` + +Required env vars are documented in the parent [README.md](../README.md). diff --git a/kits/interview-scorecard-calibrator/apps/actions/orchestrate.ts b/kits/interview-scorecard-calibrator/apps/actions/orchestrate.ts new file mode 100644 index 000000000..5fa110b30 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/actions/orchestrate.ts @@ -0,0 +1,109 @@ +"use server"; + +import { getCalibrateFlowId, getLamaticClient } from "@/lib/lamatic-client"; +import { parseScorecard, splitInterviewerNotes, type Scorecard } from "@/lib/scorecard"; + +export type CalibrateInput = { + jobTitle: string; + level: string; + rubric: string; + interviewerNotes: string; +}; + +export type CalibrateResult = { + success: boolean; + data?: { + scorecard: Scorecard; + brief: string; + raw: unknown; + }; + error?: string; +}; + +/** Parse a JSON string when possible; otherwise return the original value. */ +function parseMaybeJson(value: unknown) { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** + * Invokes the Lamatic calibrate-scorecard flow and validates the structured response. + */ +export async function calibrateScorecard( + input: CalibrateInput, +): Promise { + try { + const jobTitle = input.jobTitle.trim(); + const rubric = input.rubric.trim(); + const interviewerNotes = input.interviewerNotes.trim(); + const level = input.level.trim(); + + if (!jobTitle) throw new Error("Job title is required"); + if (!rubric) throw new Error("Competency rubric is required"); + if (!interviewerNotes) throw new Error("Interviewer notes are required"); + + const interviewerBlocks = splitInterviewerNotes(interviewerNotes); + if (interviewerBlocks.length < 2) { + throw new Error( + "Provide notes from at least two interviewers, separated by '---' or 'Interviewer N:' headings", + ); + } + + const flowId = getCalibrateFlowId(); + const lamaticClient = getLamaticClient(); + const resData = await lamaticClient.executeFlow(flowId, { + job_title: jobTitle, + level: level || "Unspecified", + rubric, + interviewer_notes: interviewerNotes, + }); + + const result = + (resData as { result?: Record })?.result || resData; + const scorecardRaw = parseMaybeJson( + (result as Record)?.scorecard ?? + (result as Record)?.answer ?? + result, + ); + const scorecard = parseScorecard(scorecardRaw); + if (!scorecard) { + throw new Error( + "Flow returned an invalid scorecard payload. Check the deployed flow output schema.", + ); + } + + const brief = + typeof (result as Record)?.brief === "string" + ? ((result as Record).brief as string) + : ""; + + return { + success: true, + data: { + scorecard, + brief, + raw: resData, + }, + }; + } catch (error) { + let errorMessage = "Unknown error occurred"; + if (error instanceof Error) { + errorMessage = error.message; + if (error.message.includes("fetch failed")) { + errorMessage = + "Network error: Unable to connect to Lamatic. Check your internet connection and API URL."; + } else if (error.message.toLowerCase().includes("api key")) { + errorMessage = "Authentication error: Please check your Lamatic API key."; + } + } + + return { + success: false, + error: errorMessage, + }; + } +} diff --git a/kits/interview-scorecard-calibrator/apps/app/globals.css b/kits/interview-scorecard-calibrator/apps/app/globals.css new file mode 100644 index 000000000..60ff96441 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/app/globals.css @@ -0,0 +1,33 @@ +@import "tailwindcss"; +@plugin "@tailwindcss/typography"; + +:root { + --background: #f4f7fb; + --foreground: #0f172a; + --panel: #ffffff; + --accent: #0f766e; + --accent-soft: #ccfbf1; + --border: #dbe3ee; + --muted: #64748b; +} + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-source-sans); + --font-display: var(--font-fraunces); +} + +body { + background: + radial-gradient(circle at top left, rgba(15, 118, 110, 0.12), transparent 32%), + radial-gradient(circle at top right, rgba(14, 165, 233, 0.1), transparent 28%), + linear-gradient(180deg, #eef5f8 0%, var(--background) 48%, #e8eef5 100%); + color: var(--foreground); + font-family: var(--font-sans), "Segoe UI", sans-serif; + min-height: 100vh; +} + +.display-font { + font-family: var(--font-display), Georgia, serif; +} diff --git a/kits/interview-scorecard-calibrator/apps/app/layout.tsx b/kits/interview-scorecard-calibrator/apps/app/layout.tsx new file mode 100644 index 000000000..76edbf06b --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/app/layout.tsx @@ -0,0 +1,35 @@ +import type { Metadata } from "next"; +import { Fraunces, Source_Sans_3 } from "next/font/google"; +import "./globals.css"; + +const display = Fraunces({ + variable: "--font-fraunces", + subsets: ["latin"], +}); + +const sans = Source_Sans_3({ + variable: "--font-source-sans", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "Interview Scorecard Calibrator", + description: + "Synthesize multi-interviewer feedback into a calibrated hiring scorecard", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + {children} + + ); +} diff --git a/kits/interview-scorecard-calibrator/apps/app/page.tsx b/kits/interview-scorecard-calibrator/apps/app/page.tsx new file mode 100644 index 000000000..b65bdfd51 --- /dev/null +++ b/kits/interview-scorecard-calibrator/apps/app/page.tsx @@ -0,0 +1,359 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; +import ReactMarkdown from "react-markdown"; +import { calibrateScorecard } from "@/actions/orchestrate"; +import { splitInterviewerNotes, type Scorecard } from "@/lib/scorecard"; + +const formSchema = z.object({ + jobTitle: z.string().trim().min(1, "Job title is required"), + level: z.string().trim().optional().default(""), + rubric: z.string().trim().min(1, "Competency rubric is required"), + interviewerNotes: z + .string() + .trim() + .min(1, "Interviewer notes are required") + .refine((value) => splitInterviewerNotes(value).length >= 2, { + message: + "Provide notes from at least two interviewers, separated by '---' or 'Interviewer N (Name):' headings", + }), +}); + +type FormValues = z.infer; + +const SAMPLE: FormValues = { + jobTitle: "Senior Backend Engineer", + level: "L5", + rubric: `System Design (High): scalable services, tradeoffs, reliability +Coding (High): correctness, clarity, edge cases +Ownership (Medium): end-to-end delivery, communication`, + interviewerNotes: `Interviewer 1 (Alice, Staff Eng): +Strong system design around caching and failover. Coding solid but a bit slow. Ownership examples were concrete. Score: Design 4, Coding 3, Ownership 4. + +--- + +Interviewer 2 (Bob, Eng Manager): +Design felt hand-wavy on consistency. Coding was clean. Concerned about stakeholder communication. Score: Design 2, Coding 4, Ownership 2.`, +}; + +function recommendationStyles(value?: string) { + const key = (value || "").toLowerCase(); + if (key === "hire") return "bg-emerald-100 text-emerald-800 border-emerald-200"; + if (key === "lean-hire") return "bg-teal-100 text-teal-800 border-teal-200"; + if (key === "lean-no") return "bg-amber-100 text-amber-900 border-amber-200"; + if (key === "no-hire") return "bg-rose-100 text-rose-800 border-rose-200"; + return "bg-slate-100 text-slate-700 border-slate-200"; +} + +export default function Page() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(""); + const [brief, setBrief] = useState(""); + const [scorecard, setScorecard] = useState(null); + + const { + register, + handleSubmit, + reset, + formState: { errors }, + } = useForm({ + resolver: zodResolver(formSchema), + defaultValues: SAMPLE, + }); + + const confidencePct = useMemo(() => { + if (typeof scorecard?.confidence !== "number") return null; + return Math.round(Math.max(0, Math.min(1, scorecard.confidence)) * 100); + }, [scorecard]); + + const onSubmit = handleSubmit(async (values) => { + setLoading(true); + setError(""); + setScorecard(null); + setBrief(""); + + try { + const response = await calibrateScorecard(values); + + if (!response.success || !response.data) { + setError(response.error || "Calibration failed"); + return; + } + + setScorecard(response.data.scorecard); + setBrief(response.data.brief || ""); + } catch (err) { + setError(err instanceof Error ? err.message : "An unexpected error occurred"); + } finally { + setLoading(false); + } + }); + + function loadSample() { + reset(SAMPLE); + setError(""); + } + + function resetResults() { + setScorecard(null); + setBrief(""); + setError(""); + } + + return ( +
+
+
+

+ AgentKit Challenge +

+

+ Interview Scorecard Calibrator +

+

+ Turn fragmented panel feedback into a calibrated competency scorecard, + disagreement map, and hiring-committee brief. +

+
+ + {!scorecard && ( +
+
+
+ + +
+ +