benjivers/jev-mcp Public Public: anyone can read this; nobody can change it.
workbenchthis machine
Files 16904894ed3e75ecfac04ccf7b61044217125dad
| 1 | // Live end-to-end tests against the real TypeSafe API. |
| 2 | // |
| 3 | // Skipped unless a credential is present in the environment, so `npm test` on a |
| 4 | // clean checkout and in CI never depends on network access or spends tokens. |
| 5 | // Run with: npm run test:e2e |
| 6 | |
| 7 | import assert from "node:assert/strict"; |
| 8 | import test from "node:test"; |
| 9 | import { payload, withClient } from "./helpers.mjs"; |
| 10 | |
| 11 | const KEY_VAR = ["TYPESAFE", "API", "KEY"].join("_"); |
| 12 | const hasKey = Boolean(process.env[KEY_VAR]); |
| 13 | const options = { skip: hasKey ? false : `set ${KEY_VAR} to run live tests` }; |
| 14 | |
| 15 | /** Forward the real credential to the server under test. */ |
| 16 | function live(fn) { |
| 17 | return withClient({ withKey: false, env: { [KEY_VAR]: process.env[KEY_VAR] } }, fn); |
| 18 | } |
| 19 | |
| 20 | test("the key works and lists at least one model", options, async () => { |
| 21 | await live(async (client) => { |
| 22 | const body = payload(await client.callTool({ name: "jev_models", arguments: {} })); |
| 23 | assert.ok(body.models.length >= 1, "expected at least one available model"); |
| 24 | }); |
| 25 | }); |
| 26 | |
| 27 | test("classify routes a support ticket to the right team", options, async () => { |
| 28 | await live(async (client) => { |
| 29 | const body = payload( |
| 30 | await client.callTool({ |
| 31 | name: "jev_classify", |
| 32 | arguments: { |
| 33 | state: "My running shoes arrived in the wrong size. Can I swap them for a size 10?", |
| 34 | question: "Which team should handle this?", |
| 35 | options: { |
| 36 | returns: "Exchanges, refunds, wrong or damaged items", |
| 37 | shipping: "Delivery status, delays, lost packages", |
| 38 | billing: "Charges, invoices, payment problems", |
| 39 | }, |
| 40 | }, |
| 41 | }), |
| 42 | ); |
| 43 | assert.equal(body.choice, "returns"); |
| 44 | assert.ok(body.confidence > 0 && body.confidence <= 1); |
| 45 | assert.ok(Object.keys(body.probabilities).includes("none")); |
| 46 | }); |
| 47 | }); |
| 48 | |
| 49 | test("check returns a high probability for a plainly urgent message", options, async () => { |
| 50 | await live(async (client) => { |
| 51 | const body = payload( |
| 52 | await client.callTool({ |
| 53 | name: "jev_check", |
| 54 | arguments: { |
| 55 | state: "Help! My payouts have been failing for 3 days and customers are complaining.", |
| 56 | question: "Does this convey urgency?", |
| 57 | }, |
| 58 | }), |
| 59 | ); |
| 60 | assert.ok(body.probability_yes > 0.5, `expected an urgent read, got ${body.probability_yes}`); |
| 61 | assert.equal(body.verdict, "yes"); |
| 62 | }); |
| 63 | }); |
| 64 | |
| 65 | test("ask answers several questions about one state in a single request", options, async () => { |
| 66 | await live(async (client) => { |
| 67 | const body = payload( |
| 68 | await client.callTool({ |
| 69 | name: "jev_ask", |
| 70 | arguments: { |
| 71 | state: "I was charged twice for order A-104. Please refund the duplicate.", |
| 72 | questions: [ |
| 73 | { id: "billing", type: "check", question: "Is this about billing?" }, |
| 74 | { id: "refund", type: "check", question: "Is the customer asking for a refund?" }, |
| 75 | { id: "anger", type: "score", question: "How frustrated is the customer?", levels: ["Calm", "Frustrated", "Very angry"] }, |
| 76 | ], |
| 77 | }, |
| 78 | }), |
| 79 | ); |
| 80 | assert.ok(body.answers.billing.noul > 0.5); |
| 81 | assert.ok(body.answers.refund.noul > 0.5); |
| 82 | assert.ok(typeof body.answers.anger.score === "number"); |
| 83 | assert.ok(body.usage.input_tokens > 0); |
| 84 | }); |
| 85 | }); |