Skip to content

Commit 54df2e3

Browse files
Fix GraphQL introspection health diagnostics (#1483)
* fix graphql introspection health diagnostics * review fixes: scrub credential values from health details, fix auth classifier, unify introspection failure classification, surface retry failures --------- Co-authored-by: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com>
1 parent f78b8ca commit 54df2e3

13 files changed

Lines changed: 927 additions & 67 deletions

File tree

.changeset/quiet-graphs-report.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"executor": patch
3+
---
4+
5+
**Fix: GraphQL connections now reject credentials when schema introspection fails and show actionable tool sync diagnostics**
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import { randomBytes } from "node:crypto";
2+
3+
import { expect } from "@effect/vitest";
4+
import { Effect } from "effect";
5+
import { composePluginApi } from "@executor-js/api/server";
6+
import { connectEmulator } from "@executor-js/emulate";
7+
import { graphqlHttpPlugin } from "@executor-js/plugin-graphql/api";
8+
import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
9+
import { variable } from "@executor-js/sdk/http-auth";
10+
11+
import { createEmulatorInstance } from "../src/emulator-instance";
12+
import { scenario } from "../src/scenario";
13+
import { Api, Browser, Target } from "../src/services";
14+
15+
const api = composePluginApi([graphqlHttpPlugin()] as const);
16+
const unique = (prefix: string): string => `${prefix}_${randomBytes(4).toString("hex")}`;
17+
18+
scenario(
19+
"GraphQL · failed introspection blocks connection creation with an actionable error",
20+
{},
21+
Effect.gen(function* () {
22+
const target = yield* Target;
23+
const browser = yield* Browser;
24+
const { client: makeApiClient } = yield* Api;
25+
const identity = yield* target.newIdentity();
26+
const client = yield* makeApiClient(api, identity);
27+
const slug = unique("graphql_health");
28+
const emulatorBaseUrl = yield* createEmulatorInstance("github", "graphql-health");
29+
const emulator = yield* Effect.promise(() =>
30+
connectEmulator({ baseUrl: emulatorBaseUrl, service: "github" }),
31+
);
32+
33+
yield* Effect.promise(() =>
34+
emulator.faults.arm({
35+
match: { method: "POST", pathPattern: "/graphql" },
36+
response: { status: 401, body: { message: "Bad credentials" } },
37+
times: 10,
38+
}),
39+
);
40+
41+
yield* client.graphql.addIntegration({
42+
payload: {
43+
endpoint: `${emulatorBaseUrl}/graphql`,
44+
slug,
45+
name: "GraphQL health",
46+
authenticationTemplate: [
47+
{
48+
slug: "header",
49+
type: "apiKey",
50+
headers: { Authorization: [variable("token")] },
51+
},
52+
],
53+
},
54+
});
55+
56+
yield* Effect.gen(function* () {
57+
yield* browser.session(identity, async ({ page, step }) => {
58+
await step("Open the connection flow", async () => {
59+
await page.goto(`/integrations/${slug}?addAccount=1&owner=org&template=header`, {
60+
waitUntil: "networkidle",
61+
});
62+
await page.getByRole("heading", { name: /Add connection · GraphQL health/ }).waitFor();
63+
});
64+
65+
await step("Submit a credential rejected during schema introspection", async () => {
66+
const dialog = page.getByRole("dialog", {
67+
name: /Add connection · GraphQL health/,
68+
});
69+
await dialog.getByRole("textbox", { name: "Authorization" }).fill("invalid-token");
70+
await dialog.getByRole("button", { name: "Continue" }).click();
71+
72+
const alert = dialog.getByRole("alert");
73+
await alert.waitFor();
74+
const message = await alert.textContent();
75+
expect(message).toContain("The endpoint rejected the credential with HTTP 401.");
76+
expect(message).toContain("Check the credential and selected authentication method.");
77+
await dialog.getByText("Step 1 of 2").waitFor();
78+
expect(
79+
await page.getByText("No connections yet").count(),
80+
"the rejected credential is not saved",
81+
).toBe(1);
82+
});
83+
});
84+
85+
// The low-level API can still import an existing credential reference
86+
// without the browser's preflight. This models connections created before
87+
// the fix and proves their failed tool sync is no longer a silent zero.
88+
yield* client.connections.create({
89+
payload: {
90+
owner: "org",
91+
name: ConnectionName.make("legacy"),
92+
integration: IntegrationSlug.make(slug),
93+
template: AuthTemplateSlug.make("header"),
94+
value: "invalid-token",
95+
},
96+
});
97+
98+
yield* browser.session(identity, async ({ page, step }) => {
99+
await step("A failed existing connection explains the empty tool catalogue", async () => {
100+
await page.goto(`/integrations/${slug}?tab=tools`, { waitUntil: "networkidle" });
101+
await page.getByText("Connection rejected", { exact: true }).first().waitFor();
102+
await page
103+
.getByText("The endpoint rejected the credential with HTTP 401.", {
104+
exact: false,
105+
})
106+
.waitFor();
107+
await page.getByRole("button", { name: "Check and sync tools" }).waitFor();
108+
});
109+
110+
await step("The account row carries the same actionable health verdict", async () => {
111+
await page.getByRole("tab", { name: "Accounts" }).click();
112+
await page.getByText("Expired", { exact: true }).waitFor();
113+
await page
114+
.getByText("Check the credential and selected authentication method.", {
115+
exact: false,
116+
})
117+
.waitFor();
118+
});
119+
});
120+
}).pipe(
121+
Effect.ensuring(
122+
Effect.all(
123+
[
124+
client.integrations
125+
.remove({ params: { slug: IntegrationSlug.make(slug) } })
126+
.pipe(Effect.ignore),
127+
Effect.promise(() => emulator.faults.clear()).pipe(Effect.ignore),
128+
],
129+
{ concurrency: "unbounded" },
130+
),
131+
),
132+
);
133+
}),
134+
);

packages/core/sdk/src/core-tools.ts

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import {
2222
type Owner,
2323
} from "./ids";
2424
import { definePlugin, tool, type StaticToolSchema } from "./plugin";
25+
import { HealthCheckResult } from "./health-check";
2526
import { ToolPolicyActionSchema } from "./policies";
2627
import type { Tool } from "./tool";
2728

@@ -76,6 +77,7 @@ const ConnectionOutput = Schema.Struct({
7677
oauthClient: Schema.NullOr(Schema.String),
7778
oauthClientOwner: Schema.NullOr(OwnerSchema),
7879
oauthScope: Schema.NullOr(Schema.String),
80+
lastHealth: Schema.NullOr(HealthCheckResult),
7981
});
8082

8183
const ConnectionsListInput = Schema.Struct({
@@ -102,6 +104,7 @@ const ConnectionListItem = Schema.Struct({
102104
oauthClientOwner: Schema.NullOr(OwnerSchema),
103105
oauthScopeCount: Schema.NullOr(Schema.Number),
104106
oauthScope: Schema.optional(Schema.NullOr(Schema.String)),
107+
lastHealth: Schema.NullOr(HealthCheckResult),
105108
});
106109
const ConnectionsListOutput = Schema.Struct({
107110
connections: Schema.Array(ConnectionListItem),
@@ -171,6 +174,7 @@ const ToolOutput = Schema.Struct({
171174
});
172175
const ConnectionsRefreshOutput = Schema.Struct({
173176
tools: Schema.Array(ToolOutput),
177+
lastHealth: Schema.NullOr(HealthCheckResult),
174178
});
175179

176180
const RemovedOutput = Schema.Struct({ removed: Schema.Boolean });
@@ -373,6 +377,7 @@ const connectionToOutput = (connection: Connection) => ({
373377
oauthClient: connection.oauthClient == null ? null : String(connection.oauthClient),
374378
oauthClientOwner: connection.oauthClientOwner ?? null,
375379
oauthScope: connection.oauthScope ?? null,
380+
lastHealth: connection.lastHealth ?? null,
376381
});
377382

378383
/** Number of space-separated grants in an `oauthScope` string, or null when
@@ -396,6 +401,7 @@ const connectionToListItem = (connection: Connection, verbose: boolean) => ({
396401
oauthClient: connection.oauthClient == null ? null : String(connection.oauthClient),
397402
oauthClientOwner: connection.oauthClientOwner ?? null,
398403
oauthScopeCount: oauthScopeCount(connection.oauthScope),
404+
lastHealth: connection.lastHealth ?? null,
399405
...(verbose ? { oauthScope: connection.oauthScope ?? null } : {}),
400406
});
401407

@@ -556,7 +562,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = {
556562
tool({
557563
name: "connections.list",
558564
description:
559-
"List saved connections (the credential for one integration). Never returns the credential value. Optionally filter by integration or owner. OAuth scopes are summarized as `oauthScopeCount` by default; pass `verbose: true` to include the full `oauthScope` grant string per connection.",
565+
"List saved connections and their last health verdict. Never returns credential values. Optionally filter by integration or owner. OAuth scopes are summarized as `oauthScopeCount` by default; pass `verbose: true` to include the full `oauthScope` grant string per connection.",
560566
inputSchema: ConnectionsListInputStd,
561567
outputSchema: ConnectionsListOutputStd,
562568
execute: (input: typeof ConnectionsListInput.Type, { ctx }) =>
@@ -627,7 +633,7 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = {
627633
tool({
628634
name: "connections.refresh",
629635
description:
630-
"Re-run an integration's tool production for a saved connection, replacing that connection's persisted tools.",
636+
"Re-run an integration's tool production for a saved connection. Returns the tools and the resulting health verdict so an empty catalog is distinguishable from a failed sync.",
631637
inputSchema: ConnectionRefInputStd,
632638
outputSchema: ConnectionsRefreshOutputStd,
633639
// Refresh replaces a connection's persisted tool set; for a mutable
@@ -636,9 +642,15 @@ export const coreToolsPlugin = definePlugin((options: CoreToolsPluginOptions = {
636642
// `sources.refresh`.
637643
annotations: { requiresApproval: true },
638644
execute: (input: typeof ConnectionRefInput.Type, { ctx }) =>
639-
Effect.map(ctx.connections.refresh(connectionRefFromInput(input)), (tools) => ({
640-
tools: tools.map(toolToOutput),
641-
})),
645+
Effect.gen(function* () {
646+
const ref = connectionRefFromInput(input);
647+
const tools = yield* ctx.connections.refresh(ref);
648+
const connection = yield* ctx.connections.get(ref);
649+
return {
650+
tools: tools.map(toolToOutput),
651+
lastHealth: connection?.lastHealth ?? null,
652+
};
653+
}),
642654
}),
643655
// removed: tools.list — the cross-connection tool catalog is an
644656
// executor-surface read, not exposed on PluginCtx.

packages/core/sdk/src/executor.test.ts

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
import { definePlugin } from "./plugin";
1616
import type { CredentialProvider } from "./provider";
1717
import { IntegrationDetectionResult } from "./types";
18-
import { makeTestExecutor } from "./testing";
18+
import { makeTestExecutor, memoryCredentialsPlugin } from "./testing";
1919
import { serveOAuthTestServer } from "./testing/oauth-test-server";
2020

2121
// removed: v1 secret browser-handoff, source.configure, case-insensitive tool-id
@@ -112,6 +112,25 @@ const demoPlugin = definePlugin(() => ({
112112
}),
113113
}))();
114114

115+
const diagnosticsPlugin = definePlugin(() => ({
116+
id: "diagnostics" as const,
117+
storage: () => ({}),
118+
resolveTools: () =>
119+
Effect.succeed({
120+
tools: [],
121+
incomplete: true,
122+
incompleteReason: "Schema introspection was rejected",
123+
}),
124+
extension: (ctx) => ({
125+
seed: () =>
126+
ctx.core.integrations.register({
127+
slug: IntegrationSlug.make("diagnostics"),
128+
description: "Diagnostics",
129+
config: {},
130+
}),
131+
}),
132+
}))();
133+
115134
const detector = (id: string, confidence: IntegrationDetectionResult["confidence"]) =>
116135
definePlugin(() => ({
117136
id,
@@ -269,6 +288,59 @@ describe("createExecutor", () => {
269288
}),
270289
);
271290

291+
it.effect("surfaces failed tool sync diagnostics through connection tools", () =>
292+
Effect.gen(function* () {
293+
const executor = yield* makeTestExecutor({
294+
plugins: [memoryCredentialsPlugin(), diagnosticsPlugin] as const,
295+
coreTools: {},
296+
});
297+
yield* executor.diagnostics.seed();
298+
299+
yield* executor.execute(
300+
ToolAddress.make("executor.coreTools.connections.create"),
301+
{
302+
owner: "org",
303+
name: "main",
304+
integration: "diagnostics",
305+
template: "none",
306+
},
307+
{ onElicitation: "accept-all" },
308+
);
309+
310+
const listed = yield* executor.execute(
311+
ToolAddress.make("executor.coreTools.connections.list"),
312+
{ integration: "diagnostics" },
313+
);
314+
expect(listed).toMatchObject({
315+
connections: [
316+
{
317+
lastHealth: {
318+
status: "degraded",
319+
detail: "Tool sync failing: Schema introspection was rejected",
320+
},
321+
},
322+
],
323+
});
324+
325+
const refreshed = yield* executor.execute(
326+
ToolAddress.make("executor.coreTools.connections.refresh"),
327+
{
328+
owner: "org",
329+
name: "main",
330+
integration: "diagnostics",
331+
},
332+
{ onElicitation: "accept-all" },
333+
);
334+
expect(refreshed).toMatchObject({
335+
tools: [],
336+
lastHealth: {
337+
status: "degraded",
338+
detail: "Tool sync failing: Schema introspection was rejected",
339+
},
340+
});
341+
}),
342+
);
343+
272344
it.effect("hands pasted credential entry to the web UI", () =>
273345
Effect.gen(function* () {
274346
const executor = yield* makeTestExecutor({

packages/plugins/graphql/src/sdk/errors.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,18 @@ export class GraphqlIntrospectionError extends Schema.TaggedErrorClass<GraphqlIn
66
"GraphqlIntrospectionError",
77
{
88
message: Schema.String,
9+
status: Schema.optional(Schema.Number),
10+
reason: Schema.optional(
11+
Schema.Literals([
12+
"network",
13+
"http",
14+
"invalid-json",
15+
"invalid-shape",
16+
"missing-schema",
17+
"graphql-errors",
18+
]),
19+
),
20+
upstreamMessage: Schema.optional(Schema.String),
921
},
1022
) {}
1123

packages/plugins/graphql/src/sdk/introspect.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ export const introspect = Effect.fn("GraphQL.introspect")(function* (
267267
() =>
268268
new GraphqlIntrospectionError({
269269
message: "Failed to reach GraphQL endpoint",
270+
reason: "network",
270271
}),
271272
),
272273
);
@@ -285,6 +286,9 @@ export const introspect = Effect.fn("GraphQL.introspect")(function* (
285286
message: upstreamMessage
286287
? `Introspection failed with status ${response.status}: ${upstreamMessage}`
287288
: `Introspection failed with status ${response.status}`,
289+
status: response.status,
290+
reason: "http",
291+
...(upstreamMessage ? { upstreamMessage } : {}),
288292
});
289293
}
290294

@@ -294,6 +298,8 @@ export const introspect = Effect.fn("GraphQL.introspect")(function* (
294298
() =>
295299
new GraphqlIntrospectionError({
296300
message: `Failed to parse introspection response as JSON`,
301+
status: response.status,
302+
reason: "invalid-json",
297303
}),
298304
),
299305
);
@@ -303,22 +309,29 @@ export const introspect = Effect.fn("GraphQL.introspect")(function* (
303309
() =>
304310
new GraphqlIntrospectionError({
305311
message: "Introspection response has an invalid shape",
312+
status: response.status,
313+
reason: "invalid-shape",
306314
}),
307315
),
308316
);
309317

310318
if (json.errors && Array.isArray(json.errors) && json.errors.length > 0) {
311-
const upstreamMessage = firstUpstreamErrorMessage(json);
319+
const upstreamMessage = upstreamTextMessage(firstUpstreamErrorMessage(json) ?? "");
312320
return yield* new GraphqlIntrospectionError({
313321
message: upstreamMessage
314322
? `Introspection returned ${json.errors.length} error(s): ${upstreamMessage}`
315323
: `Introspection returned ${json.errors.length} error(s)`,
324+
status: response.status,
325+
reason: "graphql-errors",
326+
...(upstreamMessage ? { upstreamMessage } : {}),
316327
});
317328
}
318329

319330
if (!json.data?.__schema) {
320331
return yield* new GraphqlIntrospectionError({
321332
message: "Introspection response missing __schema",
333+
status: response.status,
334+
reason: "missing-schema",
322335
});
323336
}
324337

0 commit comments

Comments
 (0)