ConusAI vendor v0.1.0-beta.114

modules/knowledge/__tests__/pdf-intake-adapters.test.ts

@@ -0,0 +1,304 @@
+// The PDF intake's two adapters (template lesson core-11). The reader hands a PDF to the platform's
+// document reader and answers its passages (`@conusai-sdk/ocr/passages`); the passage index sends one
+// inline item per passage, under the file's Blob reference plus `#p{page}.{index}` and with its page,
+// blocks and file in metadata, and tombstones exactly those references. A search hit on a passage
+// cites back into the file it came from.
+import { createMemoryDataSource } from "@conusai-kit/data"
+import { DEFAULT_FILES_POLICY, files } from "@conusai-kit/files"
+import {
+  KNOWLEDGE_LIMITS,
+  type Knowledge,
+  type KnowledgeClient,
+  KnowledgeError,
+  type KnowledgeInlineItem,
+} from "@conusai-kit/platform"
+import { fakeObjectStore } from "@conusai-kit/testing"
+import { OcrError } from "@conusai-sdk/ocr"
+import { describe, expect, it } from "vitest"
+import { platformKnowledgeSearch, platformPassageIndex } from "../adapters/platform-knowledge.js"
+import { sdkKnowledgeReader } from "../adapters/reader.js"
+import type { KnowledgePassage, KnowledgeScope } from "../core/ports.js"
+
+const scope: KnowledgeScope = {
+  level: "project",
+  actor: { userId: "system:knowledge.autoindex", system: true },
+  orgId: "01900000-0000-7000-0000-000000000001",
+  workspaceId: "01900000-0000-7000-0000-000000000002",
+  projectId: "01900000-0000-7000-0000-000000000003",
+}
+const storageKey = `files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000100`
+const pathname = `org/${scope.orgId}/${storageKey}`
+
+async function storedFile() {
+  const data = createMemoryDataSource({ entities: [files] })
+  const deps = { data, store: fakeObjectStore(), policy: DEFAULT_FILES_POLICY }
+  const row = await data
+    .forScope(scope)
+    .of(files)
+    .insert({
+      status: "stored",
+      storageKey,
+      filename: "contract.pdf",
+      contentType: "application/pdf",
+      size: 10,
+      sha256: "0".repeat(64),
+      uploadedBy: "01900000-0000-7000-0000-000000000010",
+      storedAt: "2026-11-09T19:00:00.000Z",
+      uploadId: null,
+      declaredSize: null,
+      partSize: null,
+      deletingAt: null,
+    })
+  return { deps, fileId: row.id }
+}
+
+type PassageClient = Pick<KnowledgeClient, "createCollection" | "indexInline" | "job" | "remove">
+
+function passageClient(jobError: string | null = null) {
+  const calls = {
+    collections: [] as string[],
+    batches: [] as KnowledgeInlineItem[][],
+    polled: [] as string[],
+    removed: [] as string[],
+  }
+  const client: PassageClient = {
+    createCollection: async (input) => {
+      calls.collections.push(`${input.collection}@${input.environment}`)
+      return input
+    },
+    indexInline: async (input) => {
+      calls.batches.push([...input.items])
+      return input.items.map((item, at) => ({
+        documentRef: item.documentRef,
+        jobId: `job-${calls.batches.length}-${at}`,
+      }))
+    },
+    job: async (jobId) => {
+      calls.polled.push(jobId)
+      return jobError === null
+        ? { jobId, status: "succeeded", progress: 1, generation: 1, error: null }
+        : { jobId, status: "failed", progress: null, generation: null, error: jobError }
+    },
+    remove: async (documentRef) => {
+      calls.removed.push(documentRef)
+    },
+  }
+  return { client, calls }
+}
+
+const passage = (page: number, index: number, blockIds: string[] = []): KnowledgePassage => ({
+  page,
+  index,
+  blockIds,
+  text: `page ${page}, passage ${index}`,
+})
+
+describe("the passage index", () => {
+  it("indexes one tenant item per passage under the file's reference, with its page, blocks and file in metadata", async () => {
+    const { deps, fileId } = await storedFile()
+    const { client, calls } = passageClient()
+    const index = platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 })
+
+    const documentRef = await index.reference(scope, fileId)
+    expect(documentRef).toBe(pathname)
+    await index.index(scope, {
+      collectionName: "contracts",
+      documentRef,
+      fileId,
+      passages: [passage(1, 0, ["b1", "b2"]), passage(2, 0)],
+    })
+
+    expect(calls.collections).toEqual(["contracts@production"])
+    const [batch] = calls.batches
+    expect(batch?.map((item) => item.documentRef)).toEqual([`${pathname}#p1.0`, `${pathname}#p2.0`])
+    expect(batch?.[0]).toMatchObject({
+      text: "page 1, passage 0",
+      visibility: "tenant",
+      metadata: { page: "1", block_ids: "b1,b2", file_id: fileId },
+    })
+    // A passage on a page with no blocks carries no block ids rather than an empty one.
+    expect(batch?.[1]?.metadata).toEqual({ page: "2", file_id: fileId })
+    // The tenant partition is the project's, the same one a search of the project asks for.
+    expect(new Set(batch?.map((item) => item.tenant))).toEqual(new Set([batch?.[0]?.tenant]))
+    expect(batch?.[0]?.tenant).toMatch(/^[0-9a-f]{64}$/)
+    // Every item's durable job is waited for.
+    expect([...calls.polled].sort()).toEqual(["job-1-0", "job-1-1"])
+  })
+
+  it("sends at most the service's item count per call", async () => {
+    const { deps, fileId } = await storedFile()
+    const { client, calls } = passageClient()
+    const many = Array.from({ length: KNOWLEDGE_LIMITS.inlineItems + 1 }, (_, at) => passage(1, at))
+    await platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 }).index(scope, {
+      collectionName: "contracts",
+      documentRef: pathname,
+      fileId,
+      passages: many,
+    })
+    expect(calls.batches.map((batch) => batch.length)).toEqual([KNOWLEDGE_LIMITS.inlineItems, 1])
+  })
+
+  it("records a quota refusal as the document's, not as something to retry", async () => {
+    const { deps, fileId } = await storedFile()
+    const { client } = passageClient("quota-exhausted")
+    await expect(
+      platformPassageIndex(client, deps, { environment: "production", intervalMs: 1 }).index(scope, {
+        collectionName: "contracts",
+        documentRef: pathname,
+        fileId,
+        passages: [passage(1, 0)],
+      }),
+    ).rejects.toMatchObject({ reason: "quota_exhausted", permanent: "quota_exhausted" })
+  })
+
+  it("tombstones exactly the references it indexed, without the file", async () => {
+    const { deps } = await storedFile()
+    const { client, calls } = passageClient()
+    await platformPassageIndex(client, deps, { environment: "production" }).remove(scope, {
+      documentRef: pathname,
+      passages: [
+        { page: 1, index: 0 },
+        { page: 1, index: 1 },
+        { page: 3, index: 0 },
+      ],
+    })
+    expect(calls.removed).toEqual([`${pathname}#p1.0`, `${pathname}#p1.1`, `${pathname}#p3.0`])
+  })
+
+  it("takes a passage the service never indexed as already removed", async () => {
+    const { deps } = await storedFile()
+    const { client, calls } = passageClient()
+    const index = platformPassageIndex(
+      {
+        ...client,
+        remove: async (documentRef) => {
+          calls.removed.push(documentRef)
+          if (documentRef.endsWith("#p1.1")) throw new KnowledgeError("unavailable_resource", "no such document")
+        },
+      },
+      deps,
+      { environment: "production" },
+    )
+    await index.remove(scope, {
+      documentRef: pathname,
+      passages: [
+        { page: 1, index: 0 },
+        { page: 1, index: 1 },
+        { page: 2, index: 0 },
+      ],
+    })
+    expect(calls.removed).toEqual([`${pathname}#p1.0`, `${pathname}#p1.1`, `${pathname}#p2.0`])
+  })
+})
+
+describe("a search hit on a passage", () => {
+  it("cites the file it came from, with the passage's page and first block", async () => {
+    const { deps, fileId } = await storedFile()
+    const client = {
+      search: async () => [
+        {
+          documentRef: `${pathname}#p2.0`,
+          snippet: "page 2",
+          score: 0.9,
+          citations: [],
+          chunkId: "chunk-1",
+          metadata: { page: "2", block_ids: "b3,b4", file_id: fileId },
+          blockIds: [],
+        },
+      ],
+    } as unknown as Knowledge
+    const hits = await platformKnowledgeSearch(client, deps, { environment: "production" }).search(scope, {
+      collectionName: "contracts",
+      query: "page",
+      limit: 5,
+      actorId: "01900000-0000-7000-0000-000000000010",
+    })
+    expect(hits).toEqual([
+      {
+        fileId,
+        filename: "contract.pdf",
+        collectionName: "contracts",
+        score: 0.9,
+        chunk: 1,
+        snippet: "page 2",
+        page: 2,
+        blockId: "b3",
+        chunkId: "chunk-1",
+      },
+    ])
+  })
+})
+
+describe("the reader", () => {
+  const document = {
+    schema_version: "conusai.ocr.document/v1",
+    pages: [
+      {
+        index: 0,
+        lines: [
+          { text: "Agreement", order: 0, repeated: false },
+          { text: "Signed", order: 1, repeated: false },
+        ],
+        blocks: [{ id: "p1-b1", order: 0, lines: [0, 1] }],
+      },
+    ],
+  }
+
+  it("submits the PDF once under its key and answers the read document's passages", async () => {
+    const submitted: Array<{ filename?: string; idempotencyKey?: string }> = []
+    const reader = sdkKnowledgeReader(
+      {
+        submit: (async (_bytes: unknown, options: { filename?: string; idempotencyKey?: string }) => {
+          submitted.push(options)
+          return { id: 7 }
+        }) as never,
+        wait: (async (job: unknown) => {
+          expect(job).toBe(7)
+          return document
+        }) as never,
+      },
+      { intervalMs: 1 },
+    )
+    const read = await reader.read(new Uint8Array([1]), { filename: "contract.pdf", idempotencyKey: "key-1" })
+    expect(submitted).toEqual([{ filename: "contract.pdf", idempotencyKey: "key-1" }])
+    expect(read).toEqual([{ page: 1, index: 0, blockIds: ["p1-b1"], text: "Agreement\nSigned" }])
+  })
+
+  it("answers a job the platform failed, or a document it cannot cite, as not indexable", async () => {
+    const failing = sdkKnowledgeReader({
+      submit: (async () => ({ id: 1 })) as never,
+      wait: (async () => {
+        throw new OcrError("failed", { type: "job-failed", status: 200 })
+      }) as never,
+    })
+    await expect(failing.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" })).rejects.toMatchObject({
+      permanent: "not_indexable",
+    })
+
+    const longId = {
+      ...document,
+      pages: [{ ...document.pages[0], blocks: [{ id: "x".repeat(257), order: 0, lines: [0, 1] }] }],
+    }
+    const uncitable = sdkKnowledgeReader({
+      submit: (async () => ({ id: 1 })) as never,
+      wait: (async () => longId) as never,
+    })
+    await expect(uncitable.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" })).rejects.toMatchObject(
+      {
+        permanent: "not_indexable",
+      },
+    )
+  })
+
+  it("leaves a transient failure to the queue", async () => {
+    const transient = sdkKnowledgeReader({
+      submit: (async () => {
+        throw new OcrError("unavailable", { status: 503 })
+      }) as never,
+      wait: (async () => document) as never,
+    })
+    const error = await transient.read(new Uint8Array([1]), { filename: "a.pdf", idempotencyKey: "k" }).catch((e) => e)
+    expect(error).toBeInstanceOf(OcrError)
+    expect((error as { permanent?: unknown }).permanent).toBeUndefined()
+  })
+})