310 lines
13 KiB
TypeScript
310 lines
13 KiB
TypeScript
// The automatic PDF intake (template lesson core-11): an uploaded PDF of an opted-in project is read by
|
|
// the platform's document reader and indexed passage by passage into the opted-in collection, for the
|
|
// administrator who turned it on, whose permission is checked again before each step that leaves the
|
|
// process. A project that is not opted in, or has knowledge off, reads nothing; removing the document
|
|
// tombstones every passage it indexed, even once the file itself is gone.
|
|
import type { TenantScope } from "@conusai-kit/core"
|
|
import { createMemoryDataSource } from "@conusai-kit/data"
|
|
import { DEFAULT_FILES_POLICY, files, filesProcessors, sha256Hex } from "@conusai-kit/files"
|
|
import { fakeObjectStore } from "@conusai-kit/testing"
|
|
import { describe, expect, it } from "vitest"
|
|
import {
|
|
AUTO_INDEX_JOB,
|
|
AUTO_INDEX_PROCESSOR_KEY,
|
|
autoIndex,
|
|
collections,
|
|
documents,
|
|
KNOWLEDGE_ENTITIES,
|
|
passageSets,
|
|
} from "../contract/index.js"
|
|
import { autoIndexUpload } from "../core/auto-index-job.js"
|
|
import { removeDocument } from "../core/index-job.js"
|
|
import type {
|
|
DocumentAiPlatform,
|
|
KnowledgeDeps,
|
|
KnowledgePassage,
|
|
KnowledgePassageIndex,
|
|
KnowledgeReader,
|
|
} from "../core/ports.js"
|
|
import { knowledgeModule } from "../module.js"
|
|
|
|
const ADMIN = "01900000-0000-7000-0000-000000000009"
|
|
const UPLOADER = "01900000-0000-7000-0000-000000000010"
|
|
const scope: TenantScope = {
|
|
level: "project",
|
|
actor: { userId: "system:knowledge.autoindex", system: true },
|
|
orgId: "01900000-0000-7000-0000-000000000001",
|
|
workspaceId: "01900000-0000-7000-0000-000000000002",
|
|
projectId: "01900000-0000-7000-0000-000000000003",
|
|
}
|
|
const pdf = new TextEncoder().encode("%PDF-1.7 synthetic")
|
|
const read: readonly KnowledgePassage[] = [
|
|
{ page: 1, index: 0, blockIds: ["b1", "b2"], text: "First page, first passage" },
|
|
{ page: 1, index: 1, blockIds: ["b3"], text: "First page, second passage" },
|
|
{ page: 3, index: 0, blockIds: [], text: "Third page" },
|
|
]
|
|
|
|
function makeDeps(
|
|
options: { enabled?: boolean; reader?: KnowledgeReader; indexError?: Error; enablement?: DocumentAiPlatform } = {},
|
|
) {
|
|
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
|
|
const store = fakeObjectStore()
|
|
const calls = {
|
|
reads: [] as Array<{ filename: string; idempotencyKey: string }>,
|
|
indexed: [] as Array<{ collectionName: string; fileId: string; passages: readonly KnowledgePassage[] }>,
|
|
removed: [] as Array<{ documentRef: string; passages: ReadonlyArray<{ page: number; index: number }> }>,
|
|
admitted: [] as string[],
|
|
}
|
|
const reader: KnowledgeReader = options.reader ?? {
|
|
async read(bytes, input) {
|
|
expect(bytes).toEqual(pdf)
|
|
calls.reads.push({ filename: input.filename, idempotencyKey: input.idempotencyKey })
|
|
return read
|
|
},
|
|
}
|
|
const passages: KnowledgePassageIndex = {
|
|
async reference(_scope, fileId) {
|
|
return `org/${scope.orgId}/${fileRow.storageKey}?${fileId}`
|
|
},
|
|
async index(_scope, input) {
|
|
expect(input.documentRef).toBe(`org/${scope.orgId}/${fileRow.storageKey}?${input.fileId}`)
|
|
calls.indexed.push({ collectionName: input.collectionName, fileId: input.fileId, passages: input.passages })
|
|
if (options.indexError) throw options.indexError
|
|
},
|
|
async remove(_scope, input) {
|
|
calls.removed.push({ documentRef: input.documentRef, passages: input.passages })
|
|
},
|
|
}
|
|
const deps: KnowledgeDeps = {
|
|
data,
|
|
files: { data, store, policy: DEFAULT_FILES_POLICY },
|
|
flags: { enabled: async () => options.enabled ?? true },
|
|
reader,
|
|
passages,
|
|
enablement: options.enablement ?? {
|
|
checkEligibility: async () => ({ ok: true }),
|
|
indexingQuota: async () => 1000,
|
|
},
|
|
beforeIndex: async (subject) => {
|
|
calls.admitted.push(subject)
|
|
},
|
|
}
|
|
return { data, store, deps, calls }
|
|
}
|
|
|
|
const fileRow = {
|
|
status: "stored" as const,
|
|
storageKey: `files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000100`,
|
|
filename: "contract.pdf",
|
|
contentType: "application/pdf",
|
|
size: pdf.length,
|
|
sha256: "",
|
|
uploadedBy: UPLOADER,
|
|
storedAt: "2026-11-09T19:00:00.000Z",
|
|
uploadId: null,
|
|
declaredSize: null,
|
|
partSize: null,
|
|
deletingAt: null,
|
|
}
|
|
|
|
async function seed(fixture: ReturnType<typeof makeDeps>, options: { optIn?: boolean } = {}) {
|
|
const sha256 = await sha256Hex(pdf)
|
|
await fixture.data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: ADMIN })
|
|
if (options.optIn ?? true)
|
|
await fixture.data
|
|
.forScope(scope)
|
|
.of(autoIndex)
|
|
.insert({ collectionName: "contracts", enabledBy: ADMIN, enabledAt: "2026-11-09T18:00:00.000Z" })
|
|
const file = await fixture.data
|
|
.forScope(scope)
|
|
.of(files)
|
|
.insert({ ...fileRow, sha256 })
|
|
await fixture.store.put(scope, { key: fileRow.storageKey, body: pdf, contentType: "application/pdf" })
|
|
return { fileId: file.id, storageKey: fileRow.storageKey, contentType: "application/pdf", sha256 }
|
|
}
|
|
|
|
/** The run's document, which these cases expect there to be. */
|
|
async function indexedView<T>(view: T | null): Promise<T> {
|
|
if (view === null) throw new Error("the upload made no document")
|
|
return view
|
|
}
|
|
|
|
describe("the automatic PDF intake", () => {
|
|
it("reads an opted-in project's PDF and indexes its passages into the collection, for the administrator who opted in", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
|
|
const done = await autoIndexUpload(fixture.deps, scope, upload)
|
|
|
|
expect(done).toMatchObject({ collectionName: "contracts", fileId: upload.fileId, status: "indexed" })
|
|
expect(fixture.calls.reads).toEqual([
|
|
{ filename: "contract.pdf", idempotencyKey: `knowledge.autoindex.${upload.fileId}.${upload.sha256}` },
|
|
])
|
|
expect(fixture.calls.indexed).toEqual([{ collectionName: "contracts", fileId: upload.fileId, passages: read }])
|
|
// The job runs for nobody else: the administrator's permission is what each step leaving the process checks.
|
|
expect(fixture.calls.admitted.length).toBeGreaterThan(0)
|
|
expect(new Set(fixture.calls.admitted)).toEqual(new Set([ADMIN]))
|
|
const [row] = (await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items
|
|
expect(row).toMatchObject({ filename: "contract.pdf", requestedBy: ADMIN, status: "indexed", error: null })
|
|
// What removal needs, kept beside the document: the file's reference and the passages per page.
|
|
const [set] = (await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items
|
|
expect(set).toMatchObject({
|
|
collectionName: "contracts",
|
|
fileId: upload.fileId,
|
|
documentRef: `org/${scope.orgId}/${fileRow.storageKey}?${upload.fileId}`,
|
|
pages: "2,0,1",
|
|
})
|
|
})
|
|
|
|
it("is the same work once: a redelivered job reads and indexes nothing again", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
await autoIndexUpload(fixture.deps, scope, upload)
|
|
expect(await autoIndexUpload(fixture.deps, scope, upload)).toMatchObject({ status: "indexed" })
|
|
expect(fixture.calls.reads).toHaveLength(1)
|
|
expect(fixture.calls.indexed).toHaveLength(1)
|
|
})
|
|
|
|
it("reads nothing for a project that is not opted in, has knowledge off, or a file that is not a PDF", async () => {
|
|
const notOptedIn = makeDeps()
|
|
expect(await autoIndexUpload(notOptedIn.deps, scope, await seed(notOptedIn, { optIn: false }))).toBeNull()
|
|
|
|
const off = makeDeps({ enabled: false })
|
|
expect(await autoIndexUpload(off.deps, scope, await seed(off))).toBeNull()
|
|
|
|
const text = makeDeps()
|
|
const upload = await seed(text)
|
|
expect(await autoIndexUpload(text.deps, scope, { ...upload, contentType: "text/plain" })).toBeNull()
|
|
|
|
for (const fixture of [notOptedIn, off, text]) {
|
|
expect(fixture.calls.reads).toEqual([])
|
|
expect((await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items).toEqual([])
|
|
}
|
|
})
|
|
|
|
it("reads nothing once the administrator's permission is gone: the refusal is the job's", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
const refused = Object.assign(new Error("permission_missing"), { status: 403 })
|
|
const deps: KnowledgeDeps = {
|
|
...fixture.deps,
|
|
beforeIndex: async () => {
|
|
throw refused
|
|
},
|
|
}
|
|
await expect(autoIndexUpload(deps, scope, upload)).rejects.toBe(refused)
|
|
expect(fixture.calls.reads).toEqual([])
|
|
expect(fixture.calls.indexed).toEqual([])
|
|
})
|
|
|
|
it("records a PDF the reader cannot make passages of as failed, with a stable code", async () => {
|
|
const fixture = makeDeps({
|
|
reader: {
|
|
read: async () => {
|
|
throw Object.assign(new Error("block id over 256 characters"), { permanent: "not_indexable" })
|
|
},
|
|
},
|
|
})
|
|
const upload = await seed(fixture)
|
|
expect(await autoIndexUpload(fixture.deps, scope, upload)).toMatchObject({
|
|
status: "failed",
|
|
error: "not_indexable",
|
|
})
|
|
expect(fixture.calls.indexed).toEqual([])
|
|
})
|
|
|
|
it("keeps the passage layout of an index the service refused partway, so removal still tombstones it", async () => {
|
|
const fixture = makeDeps({ indexError: Object.assign(new Error("quota"), { permanent: "quota_exhausted" }) })
|
|
const upload = await seed(fixture)
|
|
const failed = await autoIndexUpload(fixture.deps, scope, upload)
|
|
expect(failed).toMatchObject({ status: "failed", error: "quota_exhausted" })
|
|
await removeDocument(fixture.deps, scope, (await indexedView(failed)).documentId)
|
|
expect(fixture.calls.removed.map((call) => call.passages.length)).toEqual([3])
|
|
expect((await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items).toEqual([])
|
|
})
|
|
|
|
it("reads nothing once the organization is no longer eligible or its indexing quota is spent", async () => {
|
|
const ineligible = makeDeps({
|
|
enablement: {
|
|
checkEligibility: async () => ({
|
|
ok: false,
|
|
reason: "feature_disabled",
|
|
detail: "the agent_document_tools flag is off",
|
|
}),
|
|
indexingQuota: async () => 1000,
|
|
},
|
|
})
|
|
expect(await autoIndexUpload(ineligible.deps, scope, await seed(ineligible))).toMatchObject({
|
|
status: "failed",
|
|
error: "feature_disabled",
|
|
})
|
|
|
|
const spent = makeDeps({
|
|
enablement: { checkEligibility: async () => ({ ok: true }), indexingQuota: async () => 0 },
|
|
})
|
|
expect(await autoIndexUpload(spent.deps, scope, await seed(spent))).toMatchObject({
|
|
status: "failed",
|
|
error: "quota_exhausted",
|
|
})
|
|
|
|
for (const fixture of [ineligible, spent]) {
|
|
expect(fixture.calls.reads).toEqual([])
|
|
expect(fixture.calls.indexed).toEqual([])
|
|
}
|
|
})
|
|
|
|
it("refuses to run where the enablement platform is not bound, rather than skip its checks", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
const { enablement: _unbound, ...deps } = fixture.deps
|
|
await expect(autoIndexUpload(deps, scope, upload)).rejects.toThrow("enablement")
|
|
expect(fixture.calls.reads).toEqual([])
|
|
})
|
|
|
|
it("does not read a file whose bytes changed since the upload", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
expect(await autoIndexUpload(fixture.deps, scope, { ...upload, sha256: "0".repeat(64) })).toBeNull()
|
|
expect(fixture.calls.reads).toEqual([])
|
|
})
|
|
|
|
it("tombstones every passage it indexed when the document is removed, even after the file is gone", async () => {
|
|
const fixture = makeDeps()
|
|
const upload = await seed(fixture)
|
|
const done = await autoIndexUpload(fixture.deps, scope, upload)
|
|
await fixture.data.forScope(scope).of(files).delete(upload.fileId)
|
|
|
|
await removeDocument(fixture.deps, scope, (await indexedView(done)).documentId)
|
|
|
|
expect(fixture.calls.removed).toEqual([
|
|
{
|
|
documentRef: `org/${scope.orgId}/${fileRow.storageKey}?${upload.fileId}`,
|
|
passages: [
|
|
{ page: 1, index: 0 },
|
|
{ page: 1, index: 1 },
|
|
{ page: 3, index: 0 },
|
|
],
|
|
},
|
|
])
|
|
expect((await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items).toEqual([])
|
|
expect((await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items).toEqual([])
|
|
})
|
|
})
|
|
|
|
describe("the intake's place in the manifest", () => {
|
|
it("contributes a files processor for PDFs whose job the module runs", () => {
|
|
const contributions = (knowledgeModule.extends ?? []).filter((entry) => entry.point.id === filesProcessors.id)
|
|
expect(contributions.flatMap((entry) => entry.values)).toEqual([
|
|
{ key: AUTO_INDEX_PROCESSOR_KEY, contentTypes: ["application/pdf"], job: AUTO_INDEX_JOB },
|
|
])
|
|
expect(Object.keys(knowledgeModule.jobs ?? {})).toContain(AUTO_INDEX_JOB)
|
|
})
|
|
|
|
it("declares the platform operations the job uses: the knowledge service's and the document reader's", () => {
|
|
const operations = (service: string) =>
|
|
knowledgeModule.serviceRequirements?.find((requirement) => requirement.service === service)?.operations
|
|
expect(operations("knowledge")?.[AUTO_INDEX_JOB]).toEqual(["knowledge.collections:use"])
|
|
expect(operations("ocr")?.[AUTO_INDEX_JOB]).toEqual(["ocr:use"])
|
|
})
|
|
})
|