224 lines
8.7 KiB
TypeScript
224 lines
8.7 KiB
TypeScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// Actual nonowner PostgreSQL + app tenancy + Agent job. Scripted model/tools, no live provider.
|
|
import { AGENT_MODEL, AGENT_TOOLS, dataRunStore } from "@conusai-kit/agent"
|
|
import { newId } from "@conusai-kit/core"
|
|
import type { KyselyDataSource } from "@conusai-kit/data-kysely"
|
|
import { orgMemberships, TENANCY } from "@conusai-kit/tenancy"
|
|
import { beforeAll, expect, it, vi } from "vitest"
|
|
import { AGENT_SCOPE } from "../core/ports.js"
|
|
import { startRun } from "../core/runs.js"
|
|
import { type ClaimChange, claimedDataSource, claimedDelivery, terminateClaimSession } from "./claimed-job.js"
|
|
import { planFollower } from "./plan-model.js"
|
|
import { workerFixture } from "./worker-fixture.js"
|
|
|
|
let data: KyselyDataSource
|
|
beforeAll(async () => {
|
|
data = await claimedDataSource()
|
|
}, 300_000)
|
|
|
|
async function fixture(needsApproval = false) {
|
|
const { container, scope } = await workerFixture(data)
|
|
// The actual worker binding uses a system actor; stored actorId alone supplies member authority.
|
|
container.value(AGENT_SCOPE, { ...scope, actor: { userId: "system:agent", system: true } })
|
|
let models = 0
|
|
let revokeDuringModel = false
|
|
let loseDuringModel = false
|
|
let tools = 0
|
|
const adapter = planFollower()
|
|
const stream = adapter.chatStream.bind(adapter)
|
|
adapter.chatStream = async function* (...args) {
|
|
models++
|
|
if (revokeDuringModel) await removeMember()
|
|
if (loseDuringModel) await terminateClaimSession()
|
|
yield* stream(...args)
|
|
}
|
|
container.value(AGENT_MODEL, adapter)
|
|
let revoke = false
|
|
let transient = false
|
|
const removeMember = async () => {
|
|
const repos = data.forScope(scope)
|
|
const member = (await repos.of(orgMemberships).list({ limit: 10 })).items[0]
|
|
if (!member) throw new Error("expected member")
|
|
await repos.of(orgMemberships).delete(member.id)
|
|
}
|
|
container.value(AGENT_TOOLS, {
|
|
async call() {
|
|
tools++
|
|
if (transient) throw new Error("private upstream failure")
|
|
if (revoke) await removeMember()
|
|
return { status: "completed", content: [], usage: { costMicrocents: 0, steps: 1, tokens: 0 } }
|
|
},
|
|
})
|
|
const { runId } = await startRun({ data }, scope, {
|
|
goal: "Synthetic worker verification",
|
|
plan: [
|
|
{ key: "read", label: "Read", tool: "records.read", args: {}, needsApproval },
|
|
{ key: "next", label: "Next", tool: "records.next", args: {} },
|
|
],
|
|
})
|
|
const store = dataRunStore(data, scope)
|
|
return {
|
|
container,
|
|
scope,
|
|
runId,
|
|
store,
|
|
removeMember,
|
|
counts: () => ({ models, tools }),
|
|
loseDuringModel: () => {
|
|
loseDuringModel = true
|
|
},
|
|
revokeDuringModel: () => {
|
|
revokeDuringModel = true
|
|
},
|
|
revoke: () => {
|
|
revoke = true
|
|
},
|
|
fail: () => {
|
|
transient = true
|
|
},
|
|
deliver: (finalAttempt = false, change?: ClaimChange) =>
|
|
claimedDelivery(data, container, scope, runId, { finalAttempt }, change),
|
|
}
|
|
}
|
|
|
|
it("executes and replays without repeating providers through the nonowner runtime", async () => {
|
|
const job = await fixture()
|
|
await job.deliver()
|
|
expect((await job.store.get(job.runId))?.status).toBe("completed")
|
|
const counts = job.counts()
|
|
expect(counts.tools).toBe(2)
|
|
await job.deliver()
|
|
expect(job.counts()).toEqual(counts)
|
|
})
|
|
|
|
it("refuses revoked membership before the first model and after a committed step", async () => {
|
|
const before = await fixture()
|
|
await before.removeMember()
|
|
await before.deliver()
|
|
expect(before.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect((await before.store.get(before.runId))?.status).toBe("failed")
|
|
const between = await fixture()
|
|
between.revoke()
|
|
await between.deliver()
|
|
expect(between.counts()).toEqual({ models: 1, tools: 1 })
|
|
expect(await between.store.get(between.runId)).toMatchObject({ status: "failed", usageSteps: 1 })
|
|
})
|
|
|
|
it("wrong org, workspace or project cannot execute or alter the immutable stored run", async () => {
|
|
const job = await fixture()
|
|
for (const field of ["orgId", "workspaceId", "projectId"] as const) {
|
|
const container = job.container.child()
|
|
container.value(AGENT_SCOPE, { ...job.scope, [field]: newId() })
|
|
await expect(claimedDelivery(data, container, job.scope, job.runId, { finalAttempt: true })).rejects.toThrow()
|
|
}
|
|
expect(job.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect((await job.store.get(job.runId))?.status).toBe("queued")
|
|
})
|
|
|
|
it("bounded exhaustion persists terminal failure, preserving committed/replay semantics", async () => {
|
|
const job = await fixture()
|
|
job.fail()
|
|
await expect(job.deliver()).rejects.toThrow()
|
|
expect((await job.store.get(job.runId))?.status).toBe("running")
|
|
await job.deliver(true)
|
|
expect(await job.store.get(job.runId)).toMatchObject({
|
|
status: "failed",
|
|
failureDetail: "the durable job exhausted its retry attempts",
|
|
})
|
|
expect((await job.store.listSteps(job.runId))[0]?.attempts).toBe(2)
|
|
const counts = job.counts()
|
|
await job.deliver(true)
|
|
expect(job.counts()).toEqual(counts)
|
|
})
|
|
|
|
it("revocation between model and tool refuses the effect before any tool call", async () => {
|
|
const job = await fixture()
|
|
job.revokeDuringModel()
|
|
await job.deliver()
|
|
expect(job.counts()).toEqual({ models: 1, tools: 0 })
|
|
expect((await job.store.get(job.runId))?.status).toBe("failed")
|
|
})
|
|
|
|
it("approval resumes the exact call, while revocation prevents approved resumption", async () => {
|
|
const job = await fixture(true)
|
|
await job.deliver(true)
|
|
expect((await job.store.get(job.runId))?.status).toBe("awaiting_approval")
|
|
expect(job.counts().tools).toBe(0)
|
|
await job.store.decideApproval(job.runId, "read", "approved")
|
|
await job.deliver()
|
|
expect((await job.store.get(job.runId))?.status).toBe("completed")
|
|
expect(job.counts().tools).toBe(2)
|
|
await job.deliver()
|
|
expect(job.counts().tools).toBe(2)
|
|
const revoked = await fixture(true)
|
|
await revoked.deliver()
|
|
await revoked.store.decideApproval(revoked.runId, "read", "approved")
|
|
await revoked.removeMember()
|
|
const counts = revoked.counts()
|
|
await revoked.deliver()
|
|
expect(revoked.counts()).toEqual(counts)
|
|
expect((await revoked.store.get(revoked.runId))?.status).toBe("failed")
|
|
})
|
|
|
|
it("a current role demotion removes execution permission despite the retained memo", async () => {
|
|
const job = await fixture()
|
|
const repos = data.forScope(job.scope)
|
|
const member = (await repos.of(orgMemberships).list({ limit: 10 })).items[0]
|
|
if (!member) throw new Error("expected membership")
|
|
await repos.of(orgMemberships).update(member.id, { role: "viewer" }, member.version)
|
|
await job.deliver()
|
|
expect(job.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect((await job.store.get(job.runId))?.status).toBe("failed")
|
|
})
|
|
|
|
it.each(["cancel", "reclaim", "delete", "scope", "actor"] as const)(
|
|
"actual adapter refuses %s before provider or durable run mutation",
|
|
async (change) => {
|
|
const job = await fixture()
|
|
const before = await job.store.get(job.runId)
|
|
const storeCalls = vi.spyOn(data, "forScope")
|
|
await expect(job.deliver(true, change)).rejects.toMatchObject({ code: "job_claim_lost" })
|
|
expect(job.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect(storeCalls).not.toHaveBeenCalled()
|
|
storeCalls.mockRestore()
|
|
expect(await job.store.get(job.runId)).toEqual(before)
|
|
expect(await job.store.listSteps(job.runId)).toEqual([])
|
|
},
|
|
)
|
|
|
|
it("actual original attribution must match the immutable run owner", async () => {
|
|
const job = await fixture()
|
|
const before = await job.store.get(job.runId)
|
|
await expect(job.deliver(true, "owner")).rejects.toMatchObject({ code: "job_claim_lost" })
|
|
expect(job.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect(await job.store.get(job.runId)).toEqual(before)
|
|
expect(await job.store.listSteps(job.runId)).toEqual([])
|
|
})
|
|
|
|
it("actual owned guard backend loss during model admission prevents every later tool and terminal failure", async () => {
|
|
const job = await fixture()
|
|
job.loseDuringModel()
|
|
await expect(job.deliver(true)).rejects.toMatchObject({ code: "job_claim_lost" })
|
|
expect(job.counts()).toEqual({ models: 1, tools: 0 })
|
|
expect((await job.store.get(job.runId))?.status).not.toBe("failed")
|
|
})
|
|
|
|
it("actual guard loss during authorization refuses model admission before any provider call", async () => {
|
|
const job = await fixture()
|
|
const tenancy = job.container.resolve(TENANCY)
|
|
const child = job.container.child()
|
|
child.value(TENANCY, {
|
|
...tenancy,
|
|
async authorizeIn(...args) {
|
|
const result = await tenancy.authorizeIn(...args)
|
|
await terminateClaimSession()
|
|
return result
|
|
},
|
|
})
|
|
await expect(claimedDelivery(data, child, job.scope, job.runId, { finalAttempt: true })).rejects.toMatchObject({
|
|
code: "job_claim_lost",
|
|
})
|
|
expect(job.counts()).toEqual({ models: 0, tools: 0 })
|
|
expect((await job.store.get(job.runId))?.status).toBe("queued")
|
|
})
|