ConusAI vendor v0.1.0-beta.114

modules/knowledge/__tests__/documents.test.ts

@@ -0,0 +1,349 @@
+// The document requests and the index job's orchestration: idempotent (collection, file) pairs,
+// the flag gate, the content-type and size admission, the row's progress states, and which
+// failures are permanent.
+
+import type { TenantScope } from "@conusai-kit/core"
+import { createMemoryDataSource } from "@conusai-kit/data"
+import { DEFAULT_FILES_POLICY, files, sha256Hex } from "@conusai-kit/files"
+import { fakeObjectStore } from "@conusai-kit/testing"
+import { describe, expect, it } from "vitest"
+import { collections, documents, KNOWLEDGE_ENTITIES } from "../contract/index.js"
+import { listDocuments, requestIndex, requestRemove, requestSourceRemoval } from "../core/documents.js"
+import { indexDocument, removeDocument } from "../core/index-job.js"
+import type { KnowledgeDeps } from "../core/ports.js"
+import { searchKnowledge } from "../core/search.js"
+
+const USER = "01900000-0000-7000-0000-000000000009"
+const scope: TenantScope = {
+  level: "project",
+  actor: { userId: USER },
+  orgId: "01900000-0000-7000-0000-000000000001",
+  workspaceId: "01900000-0000-7000-0000-000000000002",
+  projectId: "01900000-0000-7000-0000-000000000003",
+}
+
+const fileRow = {
+  status: "stored" as const,
+  storageKey: "k/seed",
+  filename: "a.txt",
+  contentType: "text/plain",
+  size: 12,
+  sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  uploadedBy: USER,
+  storedAt: "2026-10-23T00:00:00.000Z",
+  uploadId: null,
+  declaredSize: null,
+  partSize: null,
+  deletingAt: null,
+}
+
+const makeDeps = (options: { flag?: boolean; search?: KnowledgeDeps["search"] } = {}) => {
+  const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
+  const store = fakeObjectStore()
+  const search = options.search ?? {
+    readiness: async () => ({ state: "ready" as const, reasons: [] }),
+    index: async () => undefined,
+    search: async () => [],
+    remove: async () => undefined,
+  }
+  const deps: KnowledgeDeps = {
+    data,
+    files: { data, store, policy: DEFAULT_FILES_POLICY },
+    search,
+    flags: { enabled: async () => options.flag ?? true },
+  }
+  void deps
+  return { data, deps, store }
+}
+
+const seed = async (data: ReturnType<typeof createMemoryDataSource>, contentType = "text/plain") => {
+  await data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: USER })
+  const row = await data
+    .forScope(scope)
+    .of(files)
+    .insert({ ...fileRow, ...(contentType === "text/plain" ? {} : { contentType }) })
+  return row.id
+}
+
+/** Seeds the file row AND the object the row names, the way a completed upload leaves both. */
+const seedWithBytes = async (
+  data: ReturnType<typeof createMemoryDataSource>,
+  store: ReturnType<typeof fakeObjectStore>,
+  bytes: Uint8Array,
+) => {
+  const fileId = await seed(data)
+  await data
+    .forScope(scope)
+    .of(files)
+    .update(fileId, { size: bytes.length, sha256: await sha256Hex(bytes) }, 1)
+  await store.put(scope, { key: fileRow.storageKey, body: bytes, contentType: "text/plain" })
+  return fileId
+}
+
+describe("requesting a document's indexing", () => {
+  it("creates the row, publishes the event, and a repeat request changes nothing", async () => {
+    const { data, deps } = makeDeps()
+    const fileId = await seed(data)
+    const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    expect(first.status).toBe("queued")
+    const second = await requestIndex(deps, scope, { collectionName: "contracts", fileId: first.fileId })
+    expect(second.documentId).toBe(first.documentId)
+    expect(await listDocuments(deps, scope, "contracts")).toEqual({
+      documents: [expect.objectContaining({ documentId: first.documentId, status: "queued" })],
+    })
+    // One outbox event: the subscriber's enqueue is the exactly-once path.
+    const outbox = data.outbox.pending(10)
+    expect((await outbox).filter((record) => record.name === "knowledge.documentIndexRequested")).toHaveLength(1)
+  })
+
+  it("re-queues an indexed pair instead of duplicating it", async () => {
+    const { data, deps } = makeDeps()
+    const fileId = await seed(data)
+    const first = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    await data.forScope(scope).of(documents).update(first.documentId, { status: "indexed" }, 1)
+    const again = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    expect(again.documentId).toBe(first.documentId)
+    expect(again.status).toBe("queued")
+  })
+
+  it("refuses when the flag is off, and admits text files only, within the size cap", async () => {
+    const { data, deps } = makeDeps({ flag: false })
+    const fileId = await seed(data)
+    const off = await requestIndex(deps, scope, { collectionName: "contracts", fileId }).catch((e: unknown) => e)
+    expect((off as { status?: number }).status).toBe(403)
+
+    const refusing = makeDeps()
+    const pdfFile = await seed(refusing.data, "application/pdf")
+    const pdf = await requestIndex(refusing.deps, scope, { collectionName: "contracts", fileId: pdfFile }).catch(
+      (e: unknown) => e,
+    )
+    expect((pdf as { status?: number }).status).toBe(422)
+
+    const big = makeDeps()
+    const bigFile = await seed(big.data)
+    await big.data.forScope(scope).of(files).update(bigFile, { size: 2_000_000 }, 1)
+    const tooBig = await requestIndex(big.deps, scope, { collectionName: "contracts", fileId: bigFile }).catch(
+      (e: unknown) => e,
+    )
+    expect((tooBig as { status?: number }).status).toBe(422)
+  })
+})
+
+describe("current retrieval authority", () => {
+  it("withholds cancelled and unindexed documents even while remote deletion retries", async () => {
+    const { data, deps } = makeDeps()
+    const fileId = await seed(data)
+    const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    const searching = {
+      ...deps,
+      search: {
+        ...deps.search,
+        readiness: async () => ({ state: "ready" as const, reasons: [] }),
+        index: async () => undefined,
+        remove: async () => undefined,
+        search: async () => [
+          { fileId, filename: "a.txt", collectionName: "contracts", snippet: "private source", score: 1, chunk: 1 },
+        ],
+      },
+    }
+    expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
+      [],
+    )
+    await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
+    expect(
+      (await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results,
+    ).toHaveLength(1)
+    await requestRemove(deps, scope, { collectionName: "contracts", fileId })
+    expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
+      [],
+    )
+    // A removal a person asks for names the stored file's object, read when it is asked for: if the file is deleted
+    // before the job runs, the job still removes the right text (F407, review R1).
+    const asked = (await data.outbox.pending(100)).filter((event) => event.name === "knowledge.documentRemoveRequested")
+    expect(asked).toHaveLength(1)
+    expect(asked[0]?.payload).toMatchObject({ storageKey: "k/seed" })
+  })
+
+  it("refuses a person's removal of a file being deleted, which leaves the document to the deletion's own removal", async () => {
+    const { data, deps } = makeDeps()
+    const fileId = await seed(data)
+    const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
+    const file = await data.forScope(scope).of(files).require(fileId)
+    await data
+      .forScope(scope)
+      .of(files)
+      .update(fileId, { status: "deleting", deletingAt: new Date().toISOString() }, file.version)
+    await expect(requestRemove(deps, scope, { collectionName: "contracts", fileId })).rejects.toMatchObject({
+      status: 409,
+    })
+    // Still active, so `FileDeleted` (which carries the key) cancels it and asks for the removal.
+    expect((await data.forScope(scope).of(documents).require(row.documentId)).status).toBe("indexed")
+    expect(
+      (await data.outbox.pending(100)).filter((event) => event.name === "knowledge.documentRemoveRequested"),
+    ).toEqual([])
+  })
+})
+
+describe("retained remote citations after source deletion", () => {
+  it("withholds a deleting or missing file even when its index row still says indexed", async () => {
+    const { data, deps } = makeDeps()
+    const fileId = await seed(data)
+    const row = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    await data.forScope(scope).of(documents).update(row.documentId, { status: "indexed" }, 1)
+    if (!deps.search) throw new Error("the retained citation fixture must configure search")
+    const searching = {
+      ...deps,
+      search: {
+        ...deps.search,
+        search: async () => [
+          { fileId, filename: "a.txt", collectionName: "contracts", snippet: "retained source", score: 1, chunk: 1 },
+        ],
+      },
+    }
+    expect(
+      (await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results,
+    ).toHaveLength(1)
+    await data.forScope(scope).of(files).update(fileId, { status: "deleting" }, 1)
+    expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
+      [],
+    )
+    const { storageKey } = (await data.forScope(scope).of(files).get(fileId)) ?? { storageKey: "" }
+    await data.forScope(scope).of(files).delete(fileId)
+    await requestSourceRemoval({ ...deps, flags: { enabled: () => false } }, scope, fileId, storageKey)
+    await requestSourceRemoval(deps, scope, fileId, storageKey)
+    expect((await listDocuments(deps, scope, "contracts")).documents[0]?.status).toBe("cancelled")
+    const cleanup = (await data.outbox.pending(100)).filter(
+      (event) => event.name === "knowledge.documentRemoveRequested",
+    )
+    expect(cleanup).toHaveLength(1)
+    // The deleted file's key travels with the request: its row is gone, so the removal cannot look it up (F407).
+    expect(cleanup[0]?.payload).toMatchObject({ storageKey })
+    expect((await searchKnowledge(searching, scope, { collectionName: "contracts", query: "source" })).results).toEqual(
+      [],
+    )
+  })
+})
+
+describe("the index job's orchestration", () => {
+  it("moves the row queued -> indexing -> indexed and calls the search service once", async () => {
+    const indexed: string[] = []
+    const { data, deps, store } = makeDeps({
+      search: {
+        readiness: async () => ({ state: "ready", reasons: [] }),
+        index: async (_scope, input) => {
+          indexed.push(input.fileId)
+        },
+        search: async () => [],
+        remove: async () => undefined,
+      },
+    })
+    const fileId = await seedWithBytes(data, store, new TextEncoder().encode("hello knowledge"))
+    const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    const done = await indexDocument(deps, scope, requested.documentId)
+    expect(done.status).toBe("indexed")
+    expect(indexed).toEqual([fileId])
+    // A redelivered job finds the row indexed and does the external work again never.
+    const again = await indexDocument(deps, scope, requested.documentId)
+    expect(again.status).toBe("indexed")
+    expect(indexed).toEqual([fileId])
+  })
+
+  it("records a permanent failure on the row; a transient one throws for the queue to retry", async () => {
+    const permanent = makeDeps({
+      search: {
+        readiness: async () => ({ state: "ready", reasons: [] }),
+        index: async () => {
+          throw Object.assign(new Error("not text"), { permanent: "not_text" })
+        },
+        search: async () => [],
+        remove: async () => undefined,
+      },
+    })
+    const permanentFile = await seedWithBytes(
+      permanent.data,
+      permanent.store,
+      new TextEncoder().encode("not text at all"),
+    )
+    const requested = await requestIndex(permanent.deps, scope, { collectionName: "contracts", fileId: permanentFile })
+    const failed = await indexDocument(permanent.deps, scope, requested.documentId)
+    expect(failed.status).toBe("failed")
+    expect(failed.error).toBe("not_text")
+
+    const transient = makeDeps({
+      search: {
+        readiness: async () => ({ state: "ready", reasons: [] }),
+        index: async () => {
+          throw new Error("the store hiccuped")
+        },
+        search: async () => [],
+        remove: async () => undefined,
+      },
+    })
+    const transientFile = await seedWithBytes(transient.data, transient.store, new TextEncoder().encode("hello"))
+    const requested2 = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId: transientFile })
+    await expect(indexDocument(transient.deps, scope, requested2.documentId)).rejects.toThrow("hiccuped")
+    let resumed = 0
+    const retry = {
+      ...transient.deps,
+      search: {
+        ...transient.deps.search,
+        readiness: async () => ({ state: "ready" as const, reasons: [] }),
+        search: async () => [],
+        remove: async () => undefined,
+        index: async () => {
+          resumed++
+        },
+      },
+    }
+    expect((await indexDocument(retry, scope, requested2.documentId)).status).toBe("indexed")
+    expect(resumed).toBe(1)
+  })
+
+  it("removal deletes the row and asks the service to drop the vectors", async () => {
+    const removed: string[] = []
+    const { data, deps } = makeDeps({
+      search: {
+        readiness: async () => ({ state: "ready", reasons: [] }),
+        index: async () => undefined,
+        search: async () => [],
+        remove: async (_scope, input) => {
+          removed.push(input.fileId)
+        },
+      },
+    })
+    const fileId = await seed(data)
+    const requested = await requestIndex(deps, scope, { collectionName: "contracts", fileId })
+    const cancelled = await requestRemove(deps, scope, { collectionName: "contracts", fileId })
+    expect(cancelled.status).toBe("cancelled")
+    expect((await indexDocument(deps, scope, requested.documentId)).status).toBe("cancelled")
+    // The remove job's body: delete the vectors, then the row.
+    await removeDocument(deps, scope, requested.documentId)
+    expect(removed).toEqual([fileId])
+    expect((await listDocuments(deps, scope, "contracts")).documents).toEqual([])
+  })
+})
+
+it("persists a final transient indexing failure but never overwrites cancellation on a lost claim", async () => {
+  const transient = makeDeps({
+    search: {
+      readiness: async () => ({ state: "ready", reasons: [] }),
+      index: async () => {
+        throw new Error("temporary provider outage")
+      },
+      search: async () => [],
+      remove: async () => {},
+    },
+  })
+  const fileId = await seedWithBytes(transient.data, transient.store, new TextEncoder().encode("synthetic source"))
+  const requested = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId })
+  const outcome = await indexDocument(transient.deps, scope, requested.documentId, { finalAttempt: true })
+  expect(outcome).toMatchObject({ status: "failed", error: "processing_exhausted" })
+  const controller = new AbortController()
+  controller.abort()
+  const next = await requestIndex(transient.deps, scope, { collectionName: "contracts", fileId })
+  await expect(
+    indexDocument(transient.deps, scope, next.documentId, { finalAttempt: true, signal: controller.signal }),
+  ).rejects.toBeDefined()
+  expect((await transient.data.forScope(scope).of(documents).get(next.documentId))?.status).not.toBe("failed")
+})