ConusAI vendor v0.1.0-beta.114

evals/documents/platform-items.eval.test.ts

@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The documents extraction eval on the second extractor: the platform's Document Items run (ADR-069),
+// behind the same `LineExtractor` port, scored on the same golden set, by the same scorer, against the
+// same offline thresholds as the model extractor (documents.eval.test.ts).
+//
+// Offline, the fake platform (`@conusai-kit/testing`) plays the platform: each case is a job whose
+// document is the case's OCR document, and its items run answers what the deterministic reader of
+// test/support/fake-model.ts finds on the pages the platform reads (not drawings, not empty pages, not
+// repeated lines), each item cited by its line and box. That measures this module's side of the seam:
+// the request, the mapping of `conusai.documents.items/v1` to lines, the second verification of every
+// quote against the page, the skipped pages, the scoring. The negative controls prove it can fail.
+//
+// Live there is no job to run a profile over (the golden set is text, not files), so the live items
+// eval is the customer repository's own, against its private set.
+
+import { platformItemsExtractor, sdkDocumentOcr } from "@app/modules/documents"
+import { createDocumentReader } from "@conusai-kit/platform"
+import { type FakeItemsResponder, startFakePlatform } from "@conusai-kit/testing/fake-platform"
+import { afterAll, beforeAll, describe, expect, it } from "vitest"
+import { readDocumentSchema } from "../../modules/documents/core/read-document.js"
+import { keyValueLines } from "../../test/support/fake-model.js"
+import {
+  aggregate,
+  type CaseResult,
+  failuresOf,
+  type GoldenCase,
+  loadCases,
+  loadThresholds,
+  noCache,
+  readDocumentOf,
+  report,
+  scoreCase,
+} from "./harness.js"
+
+const live = process.env.EVAL_LIVE === "1"
+const cases = loadCases()
+const thresholds = loadThresholds()
+
+/** What the platform reads off a stored document: every field of a page it does not leave out, cited by its line. */
+const deterministicItems: FakeItemsResponder = ({ document, sha256 }) => {
+  const read = readDocumentSchema.parse(document)
+  const items: Array<Record<string, unknown>> = []
+  for (const page of read.pages) {
+    if (page.class === "drawing" || page.class === "empty") continue
+    const lines = page.lines.filter((line) => !line.repeated)
+    for (const [n, field] of keyValueLines(lines.map((line) => line.text).join("\n")).entries()) {
+      const line = lines.find((candidate) => candidate.text.trim() === field.quote)
+      // The service numbers a line 1-based in reading order, the n of `p{page}-l{n}`
+      // (crates/conusai-ocr/src/items/lines.rs). A field no line holds cites no line, and its id names none.
+      const number = line ? line.order + 1 : undefined
+      items.push({
+        id: number === undefined ? `p${page.index + 1}-field${n + 1}` : `p${page.index + 1}-l${number}`,
+        section: null,
+        position: null,
+        name: field.label,
+        class: "other",
+        unit: { text: "", code: null },
+        // A key/value field's value is free text, not a quantity: the platform answers no decimal and says why.
+        quantity: { value: null, text: field.value, column: "", refusal: "not_a_number" },
+        spec: null,
+        citations: [
+          {
+            kind: "pdf_page",
+            file_sha256: sha256,
+            page: page.index + 1,
+            lines: number === undefined ? [] : [number],
+            bbox: line ? [...line.bbox] : null,
+            quote: field.quote,
+          },
+        ],
+        checks: [{ check: "quote_verbatim", status: "passed" }],
+        agreement: { status: "auto_accepted", reader: true, model: true },
+        method: "reader_and_model",
+        confidence: 0.97,
+      })
+    }
+  }
+  return items
+}
+
+describe.skipIf(live)("the platform items extractor, offline against the fake platform", () => {
+  let platform: Awaited<ReturnType<typeof startFakePlatform>>
+  const env = () => ({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" })
+
+  beforeAll(async () => {
+    platform = await startFakePlatform()
+  })
+  afterAll(async () => {
+    await platform.close()
+  })
+
+  /**
+   * Each case as a finished job of the fake platform, then the extractor over that job. Every pass submits
+   * new jobs: a job and its items run are idempotent, so a pass over the same jobs would read the runs the
+   * last pass finished, whatever the platform answers now.
+   */
+  let pass = 0
+  const evaluate = async (responder: FakeItemsResponder) => {
+    pass += 1
+    platform.ocr.setItems(responder)
+    const reader = createDocumentReader(env())
+    const extractor = platformItemsExtractor({ items: reader.items, intervalMs: 1 })
+    const ocr = sdkDocumentOcr(reader, { intervalMs: 1 })
+    const results: CaseResult[] = []
+    for (const golden of cases) {
+      platform.ocr.setDocument({ schema_version: "conusai.ocr.document/v1", ...readDocumentOf(golden) })
+      const { jobId } = await ocr.submit(new TextEncoder().encode(golden.id), {
+        filename: `${golden.id}.pdf`,
+        idempotencyKey: `items.${pass}.${golden.id}`,
+      })
+      await ocr.wait(jobId)
+      const extraction = await extractor.extract({
+        document: readDocumentOf(golden),
+        ocrJobId: jobId,
+        pageImage: async () => undefined,
+        cache: noCache,
+      })
+      results.push(scoreCase(golden, extraction))
+    }
+    return { results, metrics: aggregate(results) }
+  }
+
+  it("scores line recall, precision and quote verification above the same thresholds as the model", async () => {
+    const { results, metrics } = await evaluate(deterministicItems)
+    console.log(`platform items\n${report(results, metrics)}`)
+    expect(failuresOf(metrics, thresholds.offline)).toEqual([])
+    expect(results.flatMap((result) => result.skippedWrongly)).toEqual([])
+    expect(metrics.expected).toBeGreaterThan(30)
+    const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as { items: Array<{ profile: string }> }
+    expect(state.items.length).toBeGreaterThanOrEqual(cases.length)
+    expect(new Set(state.items.map((run) => run.profile))).toEqual(new Set(["invoice"]))
+  })
+
+  it("fails when the platform's quotes are not on the page, even when it calls them exact", async () => {
+    const { metrics } = await evaluate(async (request) =>
+      (await deterministicItems(request)).map((item) => {
+        for (const citation of item.citations as Array<{ quote: string }>)
+          citation.quote = citation.quote.replace(/\d/g, (digit) => String((Number(digit) + 1) % 10))
+        return item
+      }),
+    )
+    expect(metrics.verification).toBeLessThan(thresholds.offline.verification)
+  })
+
+  it("fails when the platform finds nothing, and when it invents items", async () => {
+    expect((await evaluate(() => [])).metrics.recall).toBe(0)
+    const { metrics } = await evaluate(async (request) => {
+      const items = await deterministicItems(request)
+      const [first] = items
+      return first
+        ? [
+            ...items,
+            {
+              ...first,
+              id: "invented",
+              name: "Invented",
+              quantity: { value: "1", text: "1", column: "", refusal: null },
+            },
+          ]
+        : items
+    })
+    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
+  })
+})
+
+/** The golden set has a case for each extractor to fail on, so neither passes by finding nothing. */
+describe("the golden set as the items extractor sees it", () => {
+  it("has fields on pages the platform reads", () => {
+    const readable = (golden: GoldenCase) =>
+      readDocumentOf(golden).pages.filter((page) => page.class !== "drawing" && page.class !== "empty").length
+    expect(cases.some((golden) => readable(golden) > 0)).toBe(true)
+  })
+})