350 lines
15 KiB
TypeScript
350 lines
15 KiB
TypeScript
// The document requests and the index job's orchestration: idempotent (collection, file) pairs,
|
|
// the flag gate, the content-type and size admission, the row's progress states, and which
|
|
// failures are permanent.
|
|
|
|
import type { TenantScope } from "@conusai-kit/core"
|
|
import { createMemoryDataSource } from "@conusai-kit/data"
|
|
import { DEFAULT_FILES_POLICY, files, sha256Hex } from "@conusai-kit/files"
|
|
import { fakeObjectStore } from "@conusai-kit/testing"
|
|
import { describe, expect, it } from "vitest"
|
|
import { collections, documents, KNOWLEDGE_ENTITIES } from "../contract/index.js"
|
|
import { listDocuments, requestIndex, requestRemove, requestSourceRemoval } from "../core/documents.js"
|
|
import { indexDocument, removeDocument } from "../core/index-job.js"
|
|
import type { KnowledgeDeps } from "../core/ports.js"
|
|
import { searchKnowledge } from "../core/search.js"
|
|
|
|
const USER = "01900000-0000-7000-0000-000000000009"
|
|
const scope: TenantScope = {
|
|
level: "project",
|
|
actor: { userId: USER },
|
|
orgId: "01900000-0000-7000-0000-000000000001",
|
|
workspaceId: "01900000-0000-7000-0000-000000000002",
|
|
projectId: "01900000-0000-7000-0000-000000000003",
|
|
}
|
|
|
|
const fileRow = {
|
|
status: "stored" as const,
|
|
storageKey: "k/seed",
|
|
filename: "a.txt",
|
|
contentType: "text/plain",
|
|
size: 12,
|
|
sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
|
uploadedBy: USER,
|
|
storedAt: "2026-10-23T00:00:00.000Z",
|
|
uploadId: null,
|
|
declaredSize: null,
|
|
partSize: null,
|
|
deletingAt: null,
|
|
}
|
|
|
|
const makeDeps = (options: { flag?: boolean; search?: KnowledgeDeps["search"] } = {}) => {
|
|
const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
|
|
const store = fakeObjectStore()
|
|
const search = options.search ?? {
|
|
readiness: async () => ({ state: "ready" as const, reasons: [] }),
|
|
index: async () => undefined,
|
|
search: async () => [],
|
|
remove: async () => undefined,
|
|
}
|
|
const deps: KnowledgeDeps = {
|
|
data,
|
|
files: { data, store, policy: DEFAULT_FILES_POLICY },
|
|
search,
|
|
flags: { enabled: async () => options.flag ?? true },
|
|
}
|
|
void deps
|
|
return { data, deps, store }
|
|
}
|
|
|
|
const seed = async (data: ReturnType<typeof createMemoryDataSource>, contentType = "text/plain") => {
|
|
await data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: USER })
|
|
const row = await data
|
|
.forScope(scope)
|
|
.of(files)
|
|
.insert({ ...fileRow, ...(contentType === "text/plain" ? {} : { contentType }) })
|
|
return row.id
|
|
}
|
|
|
|
/** Seeds the file row AND the object the row names, the way a completed upload leaves both. */
|
|
const seedWithBytes = async (
|
|
data: ReturnType<typeof createMemoryDataSource>,
|
|
store: ReturnType<typeof fakeObjectStore>,
|
|
bytes: Uint8Array,
|
|
) => {
|
|
const fileId = await seed(data)
|
|
await data
|
|
.forScope(scope)
|
|
.of(files)
|
|
.update(fileId, { size: bytes.length, sha256: await sha256Hex(bytes) }, 1)
|
|
await store.put(scope, { key: fileRow.storageKey, body: bytes, contentType: "text/plain" })
|
|
return fileId
|
|
}
|
|
|
|
describe("requesting a document's indexing", () => {
|
|
it("creates the row, publishes the event, and a repeat request changes nothing", async () => {
|
|
const { data, deps } = makeDeps()
|
|
const fileId = await seed(data)
|
|
const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
expect(first.status).toBe("queued")
|
|
const second = await requestIndex(deps, scope, { collectionName: "contracts", fileId: first.fileId })
|
|
expect(second.documentId).toBe(first.documentId)
|
|
expect(await listDocuments(deps, scope, "contracts")).toEqual({
|
|
documents: [expect.objectContaining({ documentId: first.documentId, status: "queued" })],
|
|
})
|
|
// One outbox event: the subscriber's enqueue is the exactly-once path.
|
|
const outbox = data.outbox.pending(10)
|
|
expect((await outbox).filter((record) => record.name === "knowledge.documentIndexRequested")).toHaveLength(1)
|
|
})
|
|
|
|
it("re-queues an indexed pair instead of duplicating it", async () => {
|
|
const { data, deps } = makeDeps()
|
|
const fileId = await seed(data)
|
|
const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
await data.forScope(scope).of(documents).update(first.documentId, { status: "indexed" }, 1)
|
|
const again = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
expect(again.documentId).toBe(first.documentId)
|
|
expect(again.status).toBe("queued")
|
|
})
|
|
|
|
it("refuses when the flag is off, and admits text files only, within the size cap", async () => {
|
|
const { data, deps } = makeDeps({ flag: false })
|
|
const fileId = await seed(data)
|
|
const off = await requestIndex(deps, scope, { collectionName: "contracts", fileId }).catch((e: unknown) => e)
|
|
expect((off as { status?: number }).status).toBe(403)
|
|
|
|
const refusing = makeDeps()
|
|
const pdfFile = await seed(refusing.data, "application/pdf")
|
|
const pdf = await requestIndex(refusing.deps, scope, { collectionName: "contracts", fileId: pdfFile }).catch(
|
|
(e: unknown) => e,
|
|
)
|
|
expect((pdf as { status?: number }).status).toBe(422)
|
|
|
|
const big = makeDeps()
|
|
const bigFile = await seed(big.data)
|
|
await big.data.forScope(scope).of(files).update(bigFile, { size: 2_000_000 }, 1)
|
|
const tooBig = await requestIndex(big.deps, scope, { collectionName: "contracts", fileId: bigFile }).catch(
|
|
(e: unknown) => e,
|
|
)
|
|
expect((tooBig as { status?: number }).status).toBe(422)
|
|
})
|
|
})
|
|
|
|
describe("current retrieval authority", () => {
|
|
it("withholds cancelled and unindexed documents even while remote deletion retries", async () => {
|
|
const { data, deps } = makeDeps()
|
|
const fileId = await seed(data)
|
|
const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
const searching = {
|
|
...deps,
|
|
search: {
|
|
...deps.search,
|
|
readiness: async () => ({ state: "ready" as const, reasons: [] }),
|
|
index: async () => undefined,
|
|
remove: async () => undefined,
|
|
search: async () => [
|
|
{ fileId, filename: "a.txt", collectionName: "contracts", snippet: "private source", score: 1, chunk: 1 },
|
|
],
|
|
},
|
|
}
|
|
expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
|
|
[],
|
|
)
|
|
await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
|
|
expect(
|
|
(await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results,
|
|
).toHaveLength(1)
|
|
await requestRemove(deps, scope, { collectionName: "contracts", fileId })
|
|
expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
|
|
[],
|
|
)
|
|
// A removal a person asks for names the stored file's object, read when it is asked for: if the file is deleted
|
|
// before the job runs, the job still removes the right text (F407, review R1).
|
|
const asked = (await data.outbox.pending(100)).filter((event) => event.name === "knowledge.documentRemoveRequested")
|
|
expect(asked).toHaveLength(1)
|
|
expect(asked[0]?.payload).toMatchObject({ storageKey: "k/seed" })
|
|
})
|
|
|
|
it("refuses a person's removal of a file being deleted, which leaves the document to the deletion's own removal", async () => {
|
|
const { data, deps } = makeDeps()
|
|
const fileId = await seed(data)
|
|
const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
|
|
const file = await data.forScope(scope).of(files).require(fileId)
|
|
await data
|
|
.forScope(scope)
|
|
.of(files)
|
|
.update(fileId, { status: "deleting", deletingAt: new Date().toISOString() }, file.version)
|
|
await expect(requestRemove(deps, scope, { collectionName: "contracts", fileId })).rejects.toMatchObject({
|
|
status: 409,
|
|
})
|
|
// Still active, so `FileDeleted` (which carries the key) cancels it and asks for the removal.
|
|
expect((await data.forScope(scope).of(documents).require(row.documentId)).status).toBe("indexed")
|
|
expect(
|
|
(await data.outbox.pending(100)).filter((event) => event.name === "knowledge.documentRemoveRequested"),
|
|
).toEqual([])
|
|
})
|
|
})
|
|
|
|
describe("retained remote citations after source deletion", () => {
|
|
it("withholds a deleting or missing file even when its index row still says indexed", async () => {
|
|
const { data, deps } = makeDeps()
|
|
const fileId = await seed(data)
|
|
const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
|
|
if (!deps.search) throw new Error("the retained citation fixture must configure search")
|
|
const searching = {
|
|
...deps,
|
|
search: {
|
|
...deps.search,
|
|
search: async () => [
|
|
{ fileId, filename: "a.txt", collectionName: "contracts", snippet: "retained source", score: 1, chunk: 1 },
|
|
],
|
|
},
|
|
}
|
|
expect(
|
|
(await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results,
|
|
).toHaveLength(1)
|
|
await data.forScope(scope).of(files).update(fileId, { status: "deleting" }, 1)
|
|
expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
|
|
[],
|
|
)
|
|
const { storageKey } = (await data.forScope(scope).of(files).get(fileId)) ?? { storageKey: "" }
|
|
await data.forScope(scope).of(files).delete(fileId)
|
|
await requestSourceRemoval({ ...deps, flags: { enabled: () => false } }, scope, fileId, storageKey)
|
|
await requestSourceRemoval(deps, scope, fileId, storageKey)
|
|
expect((await listDocuments(deps, scope, "contracts")).documents[0]?.status).toBe("cancelled")
|
|
const cleanup = (await data.outbox.pending(100)).filter(
|
|
(event) => event.name === "knowledge.documentRemoveRequested",
|
|
)
|
|
expect(cleanup).toHaveLength(1)
|
|
// The deleted file's key travels with the request: its row is gone, so the removal cannot look it up (F407).
|
|
expect(cleanup[0]?.payload).toMatchObject({ storageKey })
|
|
expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
|
|
[],
|
|
)
|
|
})
|
|
})
|
|
|
|
describe("the index job's orchestration", () => {
|
|
it("moves the row queued -> indexing -> indexed and calls the search service once", async () => {
|
|
const indexed: string[] = []
|
|
const { data, deps, store } = makeDeps({
|
|
search: {
|
|
readiness: async () => ({ state: "ready", reasons: [] }),
|
|
index: async (_scope, input) => {
|
|
indexed.push(input.fileId)
|
|
},
|
|
search: async () => [],
|
|
remove: async () => undefined,
|
|
},
|
|
})
|
|
const fileId = await seedWithBytes(data, store, new TextEncoder().encode("hello knowledge"))
|
|
const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
const done = await indexDocument(deps, scope, requested.documentId)
|
|
expect(done.status).toBe("indexed")
|
|
expect(indexed).toEqual([fileId])
|
|
// A redelivered job finds the row indexed and does the external work again never.
|
|
const again = await indexDocument(deps, scope, requested.documentId)
|
|
expect(again.status).toBe("indexed")
|
|
expect(indexed).toEqual([fileId])
|
|
})
|
|
|
|
it("records a permanent failure on the row; a transient one throws for the queue to retry", async () => {
|
|
const permanent = makeDeps({
|
|
search: {
|
|
readiness: async () => ({ state: "ready", reasons: [] }),
|
|
index: async () => {
|
|
throw Object.assign(new Error("not text"), { permanent: "not_text" })
|
|
},
|
|
search: async () => [],
|
|
remove: async () => undefined,
|
|
},
|
|
})
|
|
const permanentFile = await seedWithBytes(
|
|
permanent.data,
|
|
permanent.store,
|
|
new TextEncoder().encode("not text at all"),
|
|
)
|
|
const requested = await requestIndex(permanent.deps, scope, { collectionName: "contracts", fileId: permanentFile })
|
|
const failed = await indexDocument(permanent.deps, scope, requested.documentId)
|
|
expect(failed.status).toBe("failed")
|
|
expect(failed.error).toBe("not_text")
|
|
|
|
const transient = makeDeps({
|
|
search: {
|
|
readiness: async () => ({ state: "ready", reasons: [] }),
|
|
index: async () => {
|
|
throw new Error("the store hiccuped")
|
|
},
|
|
search: async () => [],
|
|
remove: async () => undefined,
|
|
},
|
|
})
|
|
const transientFile = await seedWithBytes(transient.data, transient.store, new TextEncoder().encode("hello"))
|
|
const requested2 = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId: transientFile })
|
|
await expect(indexDocument(transient.deps, scope, requested2.documentId)).rejects.toThrow("hiccuped")
|
|
let resumed = 0
|
|
const retry = {
|
|
...transient.deps,
|
|
search: {
|
|
...transient.deps.search,
|
|
readiness: async () => ({ state: "ready" as const, reasons: [] }),
|
|
search: async () => [],
|
|
remove: async () => undefined,
|
|
index: async () => {
|
|
resumed++
|
|
},
|
|
},
|
|
}
|
|
expect((await indexDocument(retry, scope, requested2.documentId)).status).toBe("indexed")
|
|
expect(resumed).toBe(1)
|
|
})
|
|
|
|
it("removal deletes the row and asks the service to drop the vectors", async () => {
|
|
const removed: string[] = []
|
|
const { data, deps } = makeDeps({
|
|
search: {
|
|
readiness: async () => ({ state: "ready", reasons: [] }),
|
|
index: async () => undefined,
|
|
search: async () => [],
|
|
remove: async (_scope, input) => {
|
|
removed.push(input.fileId)
|
|
},
|
|
},
|
|
})
|
|
const fileId = await seed(data)
|
|
const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
|
|
const cancelled = await requestRemove(deps, scope, { collectionName: "contracts", fileId })
|
|
expect(cancelled.status).toBe("cancelled")
|
|
expect((await indexDocument(deps, scope, requested.documentId)).status).toBe("cancelled")
|
|
// The remove job's body: delete the vectors, then the row.
|
|
await removeDocument(deps, scope, requested.documentId)
|
|
expect(removed).toEqual([fileId])
|
|
expect((await listDocuments(deps, scope, "contracts")).documents).toEqual([])
|
|
})
|
|
})
|
|
|
|
it("persists a final transient indexing failure but never overwrites cancellation on a lost claim", async () => {
|
|
const transient = makeDeps({
|
|
search: {
|
|
readiness: async () => ({ state: "ready", reasons: [] }),
|
|
index: async () => {
|
|
throw new Error("temporary provider outage")
|
|
},
|
|
search: async () => [],
|
|
remove: async () => {},
|
|
},
|
|
})
|
|
const fileId = await seedWithBytes(transient.data, transient.store, new TextEncoder().encode("synthetic source"))
|
|
const requested = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId })
|
|
const outcome = await indexDocument(transient.deps, scope, requested.documentId, { finalAttempt: true })
|
|
expect(outcome).toMatchObject({ status: "failed", error: "processing_exhausted" })
|
|
const controller = new AbortController()
|
|
controller.abort()
|
|
const next = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId })
|
|
await expect(
|
|
indexDocument(transient.deps, scope, next.documentId, { finalAttempt: true, signal: controller.signal }),
|
|
).rejects.toBeDefined()
|
|
expect((await transient.data.forScope(scope).of(documents).get(next.documentId))?.status).not.toBe("failed")
|
|
})
|