305 lines
11 KiB
TypeScript
305 lines
11 KiB
TypeScript
// The PDF intake's two adapters (template lesson core-11). The reader hands a PDF to the platform's
|
|
// document reader and answers its passages (`@conusai-sdk/ocr/passages`); the passage index sends one
|
|
// inline item per passage, under the file's Blob reference plus `#p{page}.{index}` and with its page,
|
|
// blocks and file in metadata, and tombstones exactly those references. A search hit on a passage
|
|
// cites back into the file it came from.
|
|
import { createMemoryDataSource } from "@conusai-kit/data"
|
|
import { DEFAULT_FILES_POLICY, files } from "@conusai-kit/files"
|
|
import {
|
|
KNOWLEDGE_LIMITS,
|
|
type Knowledge,
|
|
type KnowledgeClient,
|
|
KnowledgeError,
|
|
type KnowledgeInlineItem,
|
|
} from "@conusai-kit/platform"
|
|
import { fakeObjectStore } from "@conusai-kit/testing"
|
|
import { OcrError } from "@conusai-sdk/ocr"
|
|
import { describe, expect, it } from "vitest"
|
|
import { platformKnowledgeSearch, platformPassageIndex } from "../adapters/platform-knowledge.js"
|
|
import { sdkKnowledgeReader } from "../adapters/reader.js"
|
|
import type { KnowledgePassage, KnowledgeScope } from "../core/ports.js"
|
|
|
|
const scope: KnowledgeScope = {
|
|
level: "project",
|
|
actor: { userId: "system:knowledge.autoindex", system: true },
|
|
orgId: "01900000-0000-7000-0000-000000000001",
|
|
workspaceId: "01900000-0000-7000-0000-000000000002",
|
|
projectId: "01900000-0000-7000-0000-000000000003",
|
|
}
|
|
const storageKey = `files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000100`
|
|
const pathname = `org/${scope.orgId}/${storageKey}`
|
|
|
|
async function storedFile() {
|
|
const data = createMemoryDataSource({ entities: [files] })
|
|
const deps = { data, store: fakeObjectStore(), policy: DEFAULT_FILES_POLICY }
|
|
const row = await data
|
|
.forScope(scope)
|
|
.of(files)
|
|
.insert({
|
|
status: "stored",
|
|
storageKey,
|
|
filename: "contract.pdf",
|
|
contentType: "application/pdf",
|
|
size: 10,
|
|
sha256: "0".repeat(64),
|
|
uploadedBy: "01900000-0000-7000-0000-000000000010",
|
|
storedAt: "2026-11-09T19:00:00.000Z",
|
|
uploadId: null,
|
|
declaredSize: null,
|
|
partSize: null,
|
|
deletingAt: null,
|
|
})
|
|
return { deps, fileId: row.id }
|
|
}
|
|
|
|
type PassageClient = Pick<KnowledgeClient, "createCollection" | "indexInline" | "job" | "remove">
|
|
|
|
function passageClient(jobError: string | null = null) {
|
|
const calls = {
|
|
collections: [] as string[],
|
|
batches: [] as KnowledgeInlineItem[][],
|
|
polled: [] as string[],
|
|
removed: [] as string[],
|
|
}
|
|
const client: PassageClient = {
|
|
createCollection: async (input) => {
|
|
calls.collections.push(`${input.collection}@${input.environment}`)
|
|
return input
|
|
},
|
|
indexInline: async (input) => {
|
|
calls.batches.push([...input.items])
|
|
return input.items.map((item, at) => ({
|
|
documentRef: item.documentRef,
|
|
jobId: `job-${calls.batches.length}-${at}`,
|
|
}))
|
|
},
|
|
job: async (jobId) => {
|
|
calls.polled.push(jobId)
|
|
return jobError === null
|
|
? { jobId, status: "succeeded", progress: 1, generation: 1, error: null }
|
|
: { jobId, status: "failed", progress: null, generation: null, error: jobError }
|
|
},
|
|
remove: async (documentRef) => {
|
|
calls.removed.push(documentRef)
|
|
},
|
|
}
|
|
return { client, calls }
|
|
}
|
|
|
|
const passage = (page: number, index: number, blockIds: string[] = []): KnowledgePassage => ({
|
|
page,
|
|
index,
|
|
blockIds,
|
|
text: `page ${page}, passage ${index}`,
|
|
})
|
|
|
|
describe("the passage index", () => {
|
|
it("indexes one tenant item per passage under the file's reference, with its page, blocks and file in metadata", async () => {
|
|
const { deps, fileId } = await storedFile()
|
|
const { client, calls } = passageClient()
|
|
const index = platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 })
|
|
|
|
const documentRef = await index.reference(scope, fileId)
|
|
expect(documentRef).toBe(pathname)
|
|
await index.index(scope, {
|
|
collectionName: "contracts",
|
|
documentRef,
|
|
fileId,
|
|
passages: [passage(1, 0, ["b1", "b2"]), passage(2, 0)],
|
|
})
|
|
|
|
expect(calls.collections).toEqual(["contracts@production"])
|
|
const [batch] = calls.batches
|
|
expect(batch?.map((item) => item.documentRef)).toEqual([`${pathname}#p1.0`, `${pathname}#p2.0`])
|
|
expect(batch?.[0]).toMatchObject({
|
|
text: "page 1, passage 0",
|
|
visibility: "tenant",
|
|
metadata: { page: "1", block_ids: "b1,b2", file_id: fileId },
|
|
})
|
|
// A passage on a page with no blocks carries no block ids rather than an empty one.
|
|
expect(batch?.[1]?.metadata).toEqual({ page: "2", file_id: fileId })
|
|
// The tenant partition is the project's, the same one a search of the project asks for.
|
|
expect(new Set(batch?.map((item) => item.tenant))).toEqual(new Set([batch?.[0]?.tenant]))
|
|
expect(batch?.[0]?.tenant).toMatch(/^[0-9a-f]{64}$/)
|
|
// Every item's durable job is waited for.
|
|
expect([...calls.polled].sort()).toEqual(["job-1-0", "job-1-1"])
|
|
})
|
|
|
|
it("sends at most the service's item count per call", async () => {
|
|
const { deps, fileId } = await storedFile()
|
|
const { client, calls } = passageClient()
|
|
const many = Array.from({ length: KNOWLEDGE_LIMITS.inlineItems + 1 }, (_, at) => passage(1, at))
|
|
await platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 }).index(scope, {
|
|
collectionName: "contracts",
|
|
documentRef: pathname,
|
|
fileId,
|
|
passages: many,
|
|
})
|
|
expect(calls.batches.map((batch) => batch.length)).toEqual([KNOWLEDGE_LIMITS.inlineItems, 1])
|
|
})
|
|
|
|
it("records a quota refusal as the document's, not as something to retry", async () => {
|
|
const { deps, fileId } = await storedFile()
|
|
const { client } = passageClient("quota-exhausted")
|
|
await expect(
|
|
platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 }).index(scope, {
|
|
collectionName: "contracts",
|
|
documentRef: pathname,
|
|
fileId,
|
|
passages: [passage(1, 0)],
|
|
}),
|
|
).rejects.toMatchObject({ reason: "quota_exhausted", permanent: "quota_exhausted" })
|
|
})
|
|
|
|
it("tombstones exactly the references it indexed, without the file", async () => {
|
|
const { deps } = await storedFile()
|
|
const { client, calls } = passageClient()
|
|
await platformPassageIndex(client, deps, { environment: "production" }).remove(scope, {
|
|
documentRef: pathname,
|
|
passages: [
|
|
{ page: 1, index: 0 },
|
|
{ page: 1, index: 1 },
|
|
{ page: 3, index: 0 },
|
|
],
|
|
})
|
|
expect(calls.removed).toEqual([`${pathname}#p1.0`, `${pathname}#p1.1`, `${pathname}#p3.0`])
|
|
})
|
|
|
|
it("takes a passage the service never indexed as already removed", async () => {
|
|
const { deps } = await storedFile()
|
|
const { client, calls } = passageClient()
|
|
const index = platformPassageIndex(
|
|
{
|
|
...client,
|
|
remove: async (documentRef) => {
|
|
calls.removed.push(documentRef)
|
|
if (documentRef.endsWith("#p1.1")) throw new KnowledgeError("unavailable_resource", "no such document")
|
|
},
|
|
},
|
|
deps,
|
|
{ environment: "production" },
|
|
)
|
|
await index.remove(scope, {
|
|
documentRef: pathname,
|
|
passages: [
|
|
{ page: 1, index: 0 },
|
|
{ page: 1, index: 1 },
|
|
{ page: 2, index: 0 },
|
|
],
|
|
})
|
|
expect(calls.removed).toEqual([`${pathname}#p1.0`, `${pathname}#p1.1`, `${pathname}#p2.0`])
|
|
})
|
|
})
|
|
|
|
describe("a search hit on a passage", () => {
|
|
it("cites the file it came from, with the passage's page and first block", async () => {
|
|
const { deps, fileId } = await storedFile()
|
|
const client = {
|
|
search: async () => [
|
|
{
|
|
documentRef: `${pathname}#p2.0`,
|
|
snippet: "page 2",
|
|
score: 0.9,
|
|
citations: [],
|
|
chunkId: "chunk-1",
|
|
metadata: { page: "2", block_ids: "b3,b4", file_id: fileId },
|
|
blockIds: [],
|
|
},
|
|
],
|
|
} as unknown as Knowledge
|
|
const hits = await platformKnowledgeSearch(client, deps, { environment: "production" }).search(scope, {
|
|
collectionName: "contracts",
|
|
query: "page",
|
|
limit: 5,
|
|
actorId: "01900000-0000-7000-0000-000000000010",
|
|
})
|
|
expect(hits).toEqual([
|
|
{
|
|
fileId,
|
|
filename: "contract.pdf",
|
|
collectionName: "contracts",
|
|
score: 0.9,
|
|
chunk: 1,
|
|
snippet: "page 2",
|
|
page: 2,
|
|
blockId: "b3",
|
|
chunkId: "chunk-1",
|
|
},
|
|
])
|
|
})
|
|
})
|
|
|
|
describe("the reader", () => {
|
|
const document = {
|
|
schema_version: "conusai.ocr.document/v1",
|
|
pages: [
|
|
{
|
|
index: 0,
|
|
lines: [
|
|
{ text: "Agreement", order: 0, repeated: false },
|
|
{ text: "Signed", order: 1, repeated: false },
|
|
],
|
|
blocks: [{ id: "p1-b1", order: 0, lines: [0, 1] }],
|
|
},
|
|
],
|
|
}
|
|
|
|
it("submits the PDF once under its key and answers the read document's passages", async () => {
|
|
const submitted: Array<{ filename?: string; idempotencyKey?: string }> = []
|
|
const reader = sdkKnowledgeReader(
|
|
{
|
|
submit: (async (_bytes: unknown, options: { filename?: string; idempotencyKey?: string }) => {
|
|
submitted.push(options)
|
|
return { id: 7 }
|
|
}) as never,
|
|
wait: (async (job: unknown) => {
|
|
expect(job).toBe(7)
|
|
return document
|
|
}) as never,
|
|
},
|
|
{ intervalMs: 1 },
|
|
)
|
|
const read = await reader.read(new Uint8Array([1]), { filename: "contract.pdf", idempotencyKey: "key-1" })
|
|
expect(submitted).toEqual([{ filename: "contract.pdf", idempotencyKey: "key-1" }])
|
|
expect(read).toEqual([{ page: 1, index: 0, blockIds: ["p1-b1"], text: "Agreement\nSigned" }])
|
|
})
|
|
|
|
it("answers a job the platform failed, or a document it cannot cite, as not indexable", async () => {
|
|
const failing = sdkKnowledgeReader({
|
|
submit: (async () => ({ id: 1 })) as never,
|
|
wait: (async () => {
|
|
throw new OcrError("failed", { type: "job-failed", status: 200 })
|
|
}) as never,
|
|
})
|
|
await expect(failing.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" })).rejects.toMatchObject({
|
|
permanent: "not_indexable",
|
|
})
|
|
|
|
const longId = {
|
|
...document,
|
|
pages: [{ ...document.pages[0], blocks: [{ id: "x".repeat(257), order: 0, lines: [0, 1] }] }],
|
|
}
|
|
const uncitable = sdkKnowledgeReader({
|
|
submit: (async () => ({ id: 1 })) as never,
|
|
wait: (async () => longId) as never,
|
|
})
|
|
await expect(uncitable.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" })).rejects.toMatchObject(
|
|
{
|
|
permanent: "not_indexable",
|
|
},
|
|
)
|
|
})
|
|
|
|
it("leaves a transient failure to the queue", async () => {
|
|
const transient = sdkKnowledgeReader({
|
|
submit: (async () => {
|
|
throw new OcrError("unavailable", { status: 503 })
|
|
}) as never,
|
|
wait: (async () => document) as never,
|
|
})
|
|
const error = await transient.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" }).catch((e) => e)
|
|
expect(error).toBeInstanceOf(OcrError)
|
|
expect((error as { permanent?: unknown }).permanent).toBeUndefined()
|
|
})
|
|
})
|