@@ -0,0 +1,86 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
import { readdirSync, readFileSync } from "node:fs"
|
||||
import { generateConvex } from "@conusai-kit/data-convex"
|
||||
import { describe, expect, it, vi } from "vitest"
|
||||
|
||||
const inclusion = vi.hoisted(() => ({
|
||||
schemaVersion: 1,
|
||||
selected: [] as string[],
|
||||
everAdopted: [] as string[],
|
||||
}))
|
||||
vi.mock("../../dist/inclusion.js", () => ({
|
||||
moduleInclusion: inclusion,
|
||||
moduleSelected: (id: string) => inclusion.selected.includes(id),
|
||||
}))
|
||||
const load = async (selected: string[], adopted = selected) => {
|
||||
inclusion.selected = selected
|
||||
inclusion.everAdopted = adopted
|
||||
vi.resetModules()
|
||||
const runtime = await import("../../../convex/lib/runtime.js")
|
||||
const { accountDataOf } = await import("@conusai-kit/tenancy")
|
||||
return { ...runtime, accountShares: accountDataOf(runtime.registry) }
|
||||
}
|
||||
|
||||
describe("Convex optional inclusion projection (preview only)", () => {
|
||||
it("commits exactly the generated schema and wrappers for the selected defaults", async () => {
|
||||
const defaults = JSON.parse(readFileSync(new URL("../../inclusion.json", import.meta.url), "utf8"))
|
||||
const fresh = await load(defaults.selected, defaults.everAdopted)
|
||||
const root = new URL("../../../convex/", import.meta.url)
|
||||
const files = generateConvex(fresh.registry, {
|
||||
header: "// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0\n",
|
||||
})
|
||||
for (const file of files) expect(readFileSync(new URL(file.path, root), "utf8"), file.path).toBe(file.content)
|
||||
expect(
|
||||
readdirSync(new URL("generated/", root))
|
||||
.filter((name) => name.endsWith(".ts"))
|
||||
.sort(),
|
||||
).toEqual(
|
||||
[
|
||||
...files.filter(({ path }) => path.startsWith("generated/")).map(({ path }) => path.split("/").at(-1)),
|
||||
"procedures.ts",
|
||||
...["documents.ts", "guides.ts", "support.ts"].filter(
|
||||
(name) => !files.some(({ path }) => path === `generated/${name}`),
|
||||
),
|
||||
].sort(),
|
||||
)
|
||||
})
|
||||
it("omits fresh optional behavior and preserves original adopted schema and rights on off", async () => {
|
||||
const ids = ["documents", "guides", "support"]
|
||||
const fresh = await load([])
|
||||
// An explicitly empty selection has no callable wrappers, independently of the
|
||||
// source template's default-on assistance selection.
|
||||
const freshFiles = generateConvex(fresh.registry)
|
||||
for (const id of ids) expect(freshFiles.some(({ path }) => path === `generated/${id}.ts`)).toBe(false)
|
||||
expect(fresh.deployableModules.map(({ id }) => id)).toEqual([
|
||||
"tenancy",
|
||||
"auth",
|
||||
"files",
|
||||
"proposals",
|
||||
"notifications",
|
||||
"branding",
|
||||
"notes", // kit:notes
|
||||
"account",
|
||||
])
|
||||
expect([...fresh.registry.procedures.keys()].filter((name) => /^(documents|guides|support)\./.test(name))).toEqual(
|
||||
[],
|
||||
)
|
||||
expect(fresh.deployableModules.find(({ id }) => id === "notes")?.extends ?? []).toEqual([]) // kit:notes
|
||||
expect(fresh.convexCapabilities.actions).not.toContain("documentsActions:process")
|
||||
const on = await load(ids)
|
||||
expect(on.convexCapabilities.procedures).toContain("documents.document.list")
|
||||
const guidesRequirements = on.deployableModules.find(({ id }) => id === "guides")?.serviceRequirements ?? []
|
||||
expect(guidesRequirements.flatMap(({ operations }) => Object.keys(operations))).toContain(
|
||||
"guides.runtime.bootstrap",
|
||||
)
|
||||
expect(guidesRequirements.flatMap(({ operations }) => Object.keys(operations))).not.toContain(
|
||||
"guides.runtime.uploadFile",
|
||||
)
|
||||
expect(on.deployableModules.find(({ id }) => id === "notes")?.extends?.length).toBeGreaterThan(0) // kit:notes
|
||||
const off = await load([], ids)
|
||||
expect(off.convexCapabilities.schema).toEqual(on.convexCapabilities.schema)
|
||||
expect(off.deployableModules.filter(({ id }) => ids.includes(id)).map(({ id }) => id)).toEqual(ids)
|
||||
expect(off.convexCapabilities.procedures.filter((name) => /^(documents|guides|support)\./.test(name))).toEqual([])
|
||||
expect(off.accountShares.map(({ key }) => key)).toEqual(on.accountShares.map(({ key }) => key))
|
||||
expect(off.convexCapabilities.actions).not.toContain("documentsActions:process")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,570 @@
|
||||
{
|
||||
"schema_version": "conusai.ocr.document/v1",
|
||||
"engine": {
|
||||
"name": "fixture-engine",
|
||||
"version": "1",
|
||||
"detection_model": null,
|
||||
"recognition_model": null
|
||||
},
|
||||
"page_count": 3,
|
||||
"pages": [
|
||||
{
|
||||
"index": 0,
|
||||
"width": 1000,
|
||||
"height": 1400,
|
||||
"unit": "px",
|
||||
"dpi": 150,
|
||||
"source": "text_layer",
|
||||
"rotation": 0,
|
||||
"text": "Quarterly report\nReference: QR-2026-04\nIssued on: 2026-04-01\nThis line has no label",
|
||||
"lines": [
|
||||
{
|
||||
"text": "Quarterly report",
|
||||
"bbox": [
|
||||
100,
|
||||
80,
|
||||
500,
|
||||
120
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.99,
|
||||
"order": 0,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "Reference: QR-2026-04",
|
||||
"bbox": [
|
||||
100,
|
||||
200,
|
||||
600,
|
||||
240
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.97,
|
||||
"order": 1,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "Issued on: 2026-04-01",
|
||||
"bbox": [
|
||||
100,
|
||||
260,
|
||||
600,
|
||||
300
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.95,
|
||||
"order": 2,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "This line has no label",
|
||||
"bbox": [
|
||||
100,
|
||||
320,
|
||||
700,
|
||||
360
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 3,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
}
|
||||
],
|
||||
"blocks": [
|
||||
{
|
||||
"id": "p1-b1",
|
||||
"bbox": [
|
||||
100,
|
||||
80,
|
||||
500,
|
||||
120
|
||||
],
|
||||
"order": 0,
|
||||
"lines": [
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p1-b2",
|
||||
"bbox": [
|
||||
100,
|
||||
200,
|
||||
600,
|
||||
240
|
||||
],
|
||||
"order": 1,
|
||||
"lines": [
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p1-b3",
|
||||
"bbox": [
|
||||
100,
|
||||
260,
|
||||
600,
|
||||
300
|
||||
],
|
||||
"order": 2,
|
||||
"lines": [
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p1-b4",
|
||||
"bbox": [
|
||||
100,
|
||||
320,
|
||||
700,
|
||||
360
|
||||
],
|
||||
"order": 3,
|
||||
"lines": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"class": "text"
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"width": 1000,
|
||||
"height": 1400,
|
||||
"unit": "px",
|
||||
"dpi": 150,
|
||||
"source": "ocr",
|
||||
"rotation": 0,
|
||||
"text": "Total amount: 1 250,00 EUR\nSigned by: A. Example\nA sentence that happens to contain a colon: and then runs on for far too many words to be a label at all",
|
||||
"lines": [
|
||||
{
|
||||
"text": "Total amount: 1 250,00 EUR",
|
||||
"bbox": [
|
||||
100,
|
||||
100,
|
||||
700,
|
||||
140
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.82,
|
||||
"order": 0,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "Signed by: A. Example",
|
||||
"bbox": [
|
||||
100,
|
||||
160,
|
||||
600,
|
||||
200
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.74,
|
||||
"order": 1,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A sentence that happens to contain a colon: and then runs on for far too many words to be a label at all",
|
||||
"bbox": [
|
||||
100,
|
||||
220,
|
||||
900,
|
||||
260
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 2,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
}
|
||||
],
|
||||
"blocks": [
|
||||
{
|
||||
"id": "p2-b1",
|
||||
"bbox": [
|
||||
100,
|
||||
100,
|
||||
700,
|
||||
140
|
||||
],
|
||||
"order": 0,
|
||||
"lines": [
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p2-b2",
|
||||
"bbox": [
|
||||
100,
|
||||
160,
|
||||
600,
|
||||
200
|
||||
],
|
||||
"order": 1,
|
||||
"lines": [
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p2-b3",
|
||||
"bbox": [
|
||||
100,
|
||||
220,
|
||||
900,
|
||||
260
|
||||
],
|
||||
"order": 2,
|
||||
"lines": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"class": "scan"
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"width": 1000,
|
||||
"height": 1400,
|
||||
"unit": "px",
|
||||
"dpi": 150,
|
||||
"source": "text_layer",
|
||||
"rotation": 0,
|
||||
"class": "drawing",
|
||||
"text": "A-101 2400\nA-102 2410\nA-103 2420\nA-104 2430\nA-105 2440\nA-106 2450\nA-107 2460\nA-108 2470\nA-109 2480\nA-110 2490\nA-111 2500\nA-112 2510",
|
||||
"lines": [
|
||||
{
|
||||
"text": "A-101 2400",
|
||||
"bbox": [
|
||||
100,
|
||||
100,
|
||||
500,
|
||||
130
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 0,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-102 2410",
|
||||
"bbox": [
|
||||
100,
|
||||
140,
|
||||
500,
|
||||
170
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 1,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-103 2420",
|
||||
"bbox": [
|
||||
100,
|
||||
180,
|
||||
500,
|
||||
210
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 2,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-104 2430",
|
||||
"bbox": [
|
||||
100,
|
||||
220,
|
||||
500,
|
||||
250
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 3,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-105 2440",
|
||||
"bbox": [
|
||||
100,
|
||||
260,
|
||||
500,
|
||||
290
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 4,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-106 2450",
|
||||
"bbox": [
|
||||
100,
|
||||
300,
|
||||
500,
|
||||
330
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 5,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-107 2460",
|
||||
"bbox": [
|
||||
100,
|
||||
340,
|
||||
500,
|
||||
370
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 6,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-108 2470",
|
||||
"bbox": [
|
||||
100,
|
||||
380,
|
||||
500,
|
||||
410
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 7,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-109 2480",
|
||||
"bbox": [
|
||||
100,
|
||||
420,
|
||||
500,
|
||||
450
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 8,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-110 2490",
|
||||
"bbox": [
|
||||
100,
|
||||
460,
|
||||
500,
|
||||
490
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 9,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-111 2500",
|
||||
"bbox": [
|
||||
100,
|
||||
500,
|
||||
500,
|
||||
530
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 10,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
},
|
||||
{
|
||||
"text": "A-112 2510",
|
||||
"bbox": [
|
||||
100,
|
||||
540,
|
||||
500,
|
||||
570
|
||||
],
|
||||
"polygon": null,
|
||||
"confidence": 0.9,
|
||||
"order": 11,
|
||||
"words": [],
|
||||
"repeated": false
|
||||
}
|
||||
],
|
||||
"blocks": [
|
||||
{
|
||||
"id": "p3-b1",
|
||||
"bbox": [
|
||||
100,
|
||||
100,
|
||||
500,
|
||||
130
|
||||
],
|
||||
"order": 0,
|
||||
"lines": [
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b2",
|
||||
"bbox": [
|
||||
100,
|
||||
140,
|
||||
500,
|
||||
170
|
||||
],
|
||||
"order": 1,
|
||||
"lines": [
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b3",
|
||||
"bbox": [
|
||||
100,
|
||||
180,
|
||||
500,
|
||||
210
|
||||
],
|
||||
"order": 2,
|
||||
"lines": [
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b4",
|
||||
"bbox": [
|
||||
100,
|
||||
220,
|
||||
500,
|
||||
250
|
||||
],
|
||||
"order": 3,
|
||||
"lines": [
|
||||
3
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b5",
|
||||
"bbox": [
|
||||
100,
|
||||
260,
|
||||
500,
|
||||
290
|
||||
],
|
||||
"order": 4,
|
||||
"lines": [
|
||||
4
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b6",
|
||||
"bbox": [
|
||||
100,
|
||||
300,
|
||||
500,
|
||||
330
|
||||
],
|
||||
"order": 5,
|
||||
"lines": [
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b7",
|
||||
"bbox": [
|
||||
100,
|
||||
340,
|
||||
500,
|
||||
370
|
||||
],
|
||||
"order": 6,
|
||||
"lines": [
|
||||
6
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b8",
|
||||
"bbox": [
|
||||
100,
|
||||
380,
|
||||
500,
|
||||
410
|
||||
],
|
||||
"order": 7,
|
||||
"lines": [
|
||||
7
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b9",
|
||||
"bbox": [
|
||||
100,
|
||||
420,
|
||||
500,
|
||||
450
|
||||
],
|
||||
"order": 8,
|
||||
"lines": [
|
||||
8
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b10",
|
||||
"bbox": [
|
||||
100,
|
||||
460,
|
||||
500,
|
||||
490
|
||||
],
|
||||
"order": 9,
|
||||
"lines": [
|
||||
9
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b11",
|
||||
"bbox": [
|
||||
100,
|
||||
500,
|
||||
500,
|
||||
530
|
||||
],
|
||||
"order": 10,
|
||||
"lines": [
|
||||
10
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "p3-b12",
|
||||
"bbox": [
|
||||
100,
|
||||
540,
|
||||
500,
|
||||
570
|
||||
],
|
||||
"order": 11,
|
||||
"lines": [
|
||||
11
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// Every string a page of the module shows is a message in English and in Lithuanian (spec §10.4), the
|
||||
// Lithuanian plural forms are real ones, and no component of the module carries a string of its own.
|
||||
|
||||
import { readdirSync, readFileSync } from "node:fs"
|
||||
import { join } from "node:path"
|
||||
import { buildRegistry } from "@conusai-kit/core"
|
||||
import { createI18n, lintUiStrings } from "@conusai-kit/i18n"
|
||||
import { describe, expect, it, vi } from "vitest"
|
||||
import { appModules } from "../../index.js"
|
||||
import { exportLabels, headerLabels, listLabels, reviewLabels, uploadLabels } from "../ui/labels.js"
|
||||
|
||||
// The messages as the app has them: from the manifest, flattened and merged by the kit.
|
||||
const catalogues = buildRegistry([...appModules]).catalogues
|
||||
const en = catalogues.en?.documents ?? {}
|
||||
const lt = catalogues.lt?.documents ?? {}
|
||||
|
||||
/** The keys a builder asks for, recorded by running it with a lookup that notes each one. */
|
||||
function keysUsed(): string[] {
|
||||
const asked: string[] = []
|
||||
const t = (key: string) => {
|
||||
asked.push(key)
|
||||
return key
|
||||
}
|
||||
uploadLabels(t)
|
||||
listLabels(t)
|
||||
reviewLabels(t, { failed: "failed", aiDisclosure: "ai", pageWord: "page" })
|
||||
exportLabels(t, "failed")
|
||||
headerLabels(t)
|
||||
return asked
|
||||
}
|
||||
|
||||
describe("the documents catalogues", () => {
|
||||
it("hold the same keys in English and in Lithuanian", () => {
|
||||
expect(Object.keys(lt).sort()).toEqual(Object.keys(en).sort())
|
||||
})
|
||||
|
||||
it("hold no empty message", () => {
|
||||
for (const [key, text] of [...Object.entries(en), ...Object.entries(lt)]) expect(text.trim(), key).not.toBe("")
|
||||
})
|
||||
|
||||
it("hold every message the components' labels ask for, in both languages", () => {
|
||||
const asked = keysUsed()
|
||||
expect(asked.length).toBeGreaterThan(30)
|
||||
for (const key of asked) {
|
||||
expect(en[key as keyof typeof en], `en is missing documents.${key}`).toBeTruthy()
|
||||
expect(lt[key as keyof typeof lt], `lt is missing documents.${key}`).toBeTruthy()
|
||||
}
|
||||
})
|
||||
|
||||
it("hold no message nothing asks for, so a renamed label does not leave a dead string behind", () => {
|
||||
// The pages ask for these directly (a heading, a sentence), not through a label builder.
|
||||
const direct = new Set(["title", "intro", "review.title", "review.back"])
|
||||
const asked = new Set([...keysUsed(), ...direct])
|
||||
expect(Object.keys(en).filter((key) => !asked.has(key))).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe("the Lithuanian plural forms", () => {
|
||||
const count = (locale: "en" | "lt", n: number) =>
|
||||
createI18n(locale, { en: { documents: en }, lt: { documents: lt } }).message("documents.review.pendingCount", {
|
||||
count: n,
|
||||
})
|
||||
|
||||
it("select one, few and other the way Lithuanian counts", () => {
|
||||
expect(count("lt", 1)).toBe("1 eilutė peržiūrai")
|
||||
expect(count("lt", 2)).toBe("2 eilutės peržiūrai")
|
||||
expect(count("lt", 9)).toBe("9 eilutės peržiūrai")
|
||||
expect(count("lt", 10)).toBe("10 eilučių peržiūrai")
|
||||
expect(count("lt", 21)).toBe("21 eilutė peržiūrai")
|
||||
})
|
||||
|
||||
it("keep English to its two forms", () => {
|
||||
expect(count("en", 1)).toBe("1 line to review")
|
||||
expect(count("en", 2)).toBe("2 lines to review")
|
||||
expect(lt["review.pendingCount"]).toContain("few {")
|
||||
expect(en["review.pendingCount"]).not.toContain("few {")
|
||||
})
|
||||
})
|
||||
|
||||
describe("the components", () => {
|
||||
const dir = new URL("../ui/", import.meta.url).pathname
|
||||
const sources = readdirSync(dir)
|
||||
.filter((name) => name.endsWith(".tsx"))
|
||||
.map((name) => ({ name, text: readFileSync(join(dir, name), "utf8") }))
|
||||
|
||||
it("carry no hard-coded user-facing string", () => {
|
||||
expect(sources.length).toBeGreaterThanOrEqual(4)
|
||||
for (const { name, text } of sources) expect(lintUiStrings(text, name), name).toEqual([])
|
||||
})
|
||||
|
||||
it("carry no colour literal: the brand tokens are the only source of colour", () => {
|
||||
for (const { name, text } of sources)
|
||||
expect(text, name).not.toMatch(/#[0-9a-fA-F]{3,8}\b|\brgb\(|\bhsl\(|\boklch\(/)
|
||||
})
|
||||
})
|
||||
|
||||
// Explicit synthetic admission: module-local fixtures exercise the selected module.
|
||||
vi.mock("../../inclusion.json", () => ({
|
||||
default: { schemaVersion: 1, selected: ["documents"], everAdopted: ["documents"] },
|
||||
}))
|
||||
@@ -0,0 +1,146 @@
|
||||
// The extractor over a scripted ModelGateway: the mapping of the gateway's failures to the codes a
|
||||
// document records, and the retry each one gets. The whole path through the real gateway client is
|
||||
// proved by test/documents-extraction.test.ts.
|
||||
|
||||
import { AiError, type GenerateObjectOptions, type ModelGateway, ModelGatewayError } from "@conusai-kit/platform"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { modelLineExtractor } from "../adapters/model-extractor.js"
|
||||
import { ExtractionFailed, type ExtractionInput, type ReadDocument } from "../core/ports.js"
|
||||
import { noCache, type StageCache } from "../core/stage-cache.js"
|
||||
|
||||
const document: ReadDocument = {
|
||||
engine: { name: "fixture", version: "1" },
|
||||
page_count: 1,
|
||||
pages: [
|
||||
{
|
||||
index: 0,
|
||||
width: 100,
|
||||
height: 100,
|
||||
source: "text_layer",
|
||||
text: "Reference: QR-1",
|
||||
class: "text",
|
||||
lines: [{ text: "Reference: QR-1", bbox: [10, 10, 60, 20], confidence: 0.9, order: 0, repeated: false }],
|
||||
blocks: [{ id: "p1-b1", bbox: [10, 10, 60, 20], order: 0, lines: [0] }],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const input = (cache: StageCache = noCache): ExtractionInput => ({ document, pageImage: async () => undefined, cache })
|
||||
|
||||
/** A gateway that answers from a script, one entry per call: a value is the reply, an Error is thrown. */
|
||||
function scripted(...script: Array<unknown>): { gateway: ModelGateway; options: GenerateObjectOptions[] } {
|
||||
const options: GenerateObjectOptions[] = []
|
||||
const gateway: ModelGateway = {
|
||||
async generateObject(_schema, _messages, call) {
|
||||
options.push(call ?? {})
|
||||
const next = script.shift()
|
||||
if (next instanceof Error) throw next
|
||||
return { value: next, model: "m" } as never
|
||||
},
|
||||
}
|
||||
return { gateway, options }
|
||||
}
|
||||
|
||||
const good = { lines: [{ label: "Reference", value: "QR-1", quote: "Reference: QR-1" }] }
|
||||
const extractorOf = (gateway: ModelGateway) => modelLineExtractor({ gateway, model: "m" })
|
||||
|
||||
describe("the model extractor", () => {
|
||||
it("names the model and the prompt version, which every proposal records", () => {
|
||||
const extractor = extractorOf(scripted().gateway)
|
||||
expect(extractor.name).toBe("m")
|
||||
expect(extractor.version).toMatch(/^[a-z-]+\.v\d+$/)
|
||||
})
|
||||
|
||||
it.each([
|
||||
["refusal", "extraction_refused", 2],
|
||||
["truncated", "extraction_truncated", 2],
|
||||
["empty", "extraction_invalid", 2],
|
||||
// a schema failure was already asked again by the gateway: the extractor does not ask a third time
|
||||
["schema", "extraction_invalid", 1],
|
||||
] as const)("a %s is asked again as the rules say, then ends as %s", async (kind, code, calls) => {
|
||||
const failure = () => new ModelGatewayError(kind, "x")
|
||||
const { gateway, options } = scripted(failure(), failure())
|
||||
await expect(extractorOf(gateway).extract(input())).rejects.toMatchObject({ code })
|
||||
expect(options).toHaveLength(calls)
|
||||
})
|
||||
|
||||
it("answers when the second attempt is good, giving a truncated reply twice the room", async () => {
|
||||
const { gateway, options } = scripted(new ModelGatewayError("truncated", "x"), good)
|
||||
const { lines } = await extractorOf(gateway).extract(input())
|
||||
expect(lines.map((line) => line.value)).toEqual(["QR-1"])
|
||||
expect(options.map((call) => call.maxTokens)).toEqual([4096, 8192])
|
||||
})
|
||||
|
||||
it("treats a request the gateway will never accept as the document's failure", async () => {
|
||||
const bad = () => new AiError("bad image", { status: 400 })
|
||||
const { gateway } = scripted(bad(), bad())
|
||||
await expect(extractorOf(gateway).extract(input())).rejects.toMatchObject({
|
||||
code: "extraction_invalid",
|
||||
name: ExtractionFailed.name,
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
["a network error", new AiError("down", { code: "network_error" })],
|
||||
["a 503", new AiError("busy", { status: 503 })],
|
||||
["a missing token scope", new AiError("no", { status: 403 })],
|
||||
["a missing model", new ModelGatewayError("no_model", "x")],
|
||||
])("leaves %s to the queue, which retries the job", async (_name, error) => {
|
||||
const { gateway, options } = scripted(error)
|
||||
await expect(extractorOf(gateway).extract(input())).rejects.toBe(error)
|
||||
expect(options).toHaveLength(1)
|
||||
})
|
||||
|
||||
it("reads a cached reply without asking the model", async () => {
|
||||
const stored = new Map<string, unknown>()
|
||||
const cache: StageCache = {
|
||||
async get(stage, key, schema) {
|
||||
const value = stored.get(`${stage}/${key}`)
|
||||
return value === undefined ? undefined : schema.parse(value)
|
||||
},
|
||||
async put(stage, key, value) {
|
||||
stored.set(`${stage}/${key}`, value)
|
||||
},
|
||||
}
|
||||
const { gateway, options } = scripted(good)
|
||||
await extractorOf(gateway).extract(input(cache))
|
||||
await extractorOf(gateway).extract(input(cache))
|
||||
expect(options).toHaveLength(1)
|
||||
// another model is another key
|
||||
const other = scripted(good)
|
||||
await modelLineExtractor({ gateway: other.gateway, model: "other" }).extract(input(cache))
|
||||
expect(other.options).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe("actual model provenance", () => {
|
||||
it("keeps the model returned by a gateway fallback and invalidates replies on an OCR engine change", async () => {
|
||||
const values = new Map<string, unknown>()
|
||||
const cache: StageCache = {
|
||||
async get(stage, key, schema) {
|
||||
const parsed = schema.safeParse(values.get(`${stage}:${key}`))
|
||||
return parsed.success ? parsed.data : undefined
|
||||
},
|
||||
async put(stage, key, value) {
|
||||
values.set(`${stage}:${key}`, value)
|
||||
},
|
||||
}
|
||||
let calls = 0
|
||||
const gateway: ModelGateway = {
|
||||
async generateObject() {
|
||||
calls++
|
||||
return { value: good, model: "actual-fallback-model" } as never
|
||||
},
|
||||
}
|
||||
const extractor = extractorOf(gateway)
|
||||
const first = await extractor.extract(input(cache))
|
||||
expect(first.lines[0]?.model).toBe("actual-fallback-model")
|
||||
expect((await extractor.extract(input(cache))).lines[0]?.model).toBe("actual-fallback-model")
|
||||
expect(calls).toBe(1)
|
||||
await extractor.extract({
|
||||
...input(cache),
|
||||
document: { ...document, engine: { ...document.engine, version: "2" } },
|
||||
})
|
||||
expect(calls).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,117 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// Disposable real MongoDB evidence; ACCOUNT_TEST_MONGODB_URL must name a replica set.
|
||||
// Each run owns a fresh database and removes only that database afterwards.
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { readFileSync } from "node:fs"
|
||||
import { dialectModuleMigrationFiles } from "@app/config"
|
||||
import { kernelModules } from "@app/modules"
|
||||
import { buildRegistry, newId } from "@conusai-kit/core"
|
||||
import {
|
||||
connectMongoDataSource,
|
||||
type MongoCommand,
|
||||
mongoCommandsMigration,
|
||||
runMongoMigrations,
|
||||
type StoredDoc,
|
||||
} from "@conusai-kit/data-mongo"
|
||||
import { makeTenant } from "@conusai-kit/testing"
|
||||
import { MongoClient } from "mongodb"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { DOCUMENTS_ENTITIES, documents, documentsModule, FAILURE_CODES } from "../index.js"
|
||||
|
||||
const url = process.env.ACCOUNT_TEST_MONGODB_URL
|
||||
const migration = (name: string) =>
|
||||
mongoCommandsMigration(
|
||||
`documents/${name}`,
|
||||
JSON.parse(
|
||||
readFileSync(new URL(`../adapters/mongo/migrations/${name}.json`, import.meta.url), "utf8"),
|
||||
) as MongoCommand[],
|
||||
)
|
||||
|
||||
describe.skipIf(!url)("Documents forward MongoDB validator evolution", () => {
|
||||
it("preserves existing rows and indexes, persists exhaustion, and retains enum and tenant refusals", async () => {
|
||||
const client = await new MongoClient(url ?? "").connect()
|
||||
const database = `documents_forward_${randomUUID().replaceAll("-", "")}`
|
||||
const db = client.db(database)
|
||||
const data = await connectMongoDataSource(client, { database, entities: DOCUMENTS_ENTITIES })
|
||||
try {
|
||||
const discovered = await dialectModuleMigrationFiles(
|
||||
buildRegistry([...kernelModules, documentsModule]),
|
||||
"mongodb",
|
||||
)
|
||||
expect(discovered.filter(({ name }) => name.startsWith("documents/")).map(({ name }) => name)).toEqual([
|
||||
"documents/0001_documents",
|
||||
"documents/0002_skipped_pages",
|
||||
"documents/0003_nullable_failure_code",
|
||||
"documents/0004_processing_exhausted",
|
||||
])
|
||||
const history = ["0001_documents", "0002_skipped_pages", "0003_nullable_failure_code"].map(migration)
|
||||
await runMongoMigrations(db, history)
|
||||
const tenant = makeTenant()
|
||||
const repo = data.forScope(tenant.project).of(documents)
|
||||
const input = {
|
||||
fileId: newId(),
|
||||
filename: "synthetic.pdf",
|
||||
status: "processing" as const,
|
||||
ocrJobId: null,
|
||||
pageCount: null,
|
||||
proposedCount: null,
|
||||
failureCode: null,
|
||||
skippedPages: null,
|
||||
finishedAt: null,
|
||||
}
|
||||
const row = await repo.insert(input)
|
||||
const rows = db.collection<StoredDoc>("documents_document")
|
||||
const before = await rows.findOne({ _id: row.id })
|
||||
const indexes = await rows.indexes()
|
||||
const [collectionBefore] = await db.listCollections({ name: "documents_document" }).toArray()
|
||||
const terminal = { status: "failed" as const, failureCode: "processing_exhausted" as const }
|
||||
// The current typed model accepts this transition, but the applied historical validator refuses it.
|
||||
await expect(repo.update(row.id, terminal, row.version)).rejects.toThrow(/[Dd]ocument failed validation/)
|
||||
expect(await rows.findOne({ _id: row.id })).toEqual(before)
|
||||
const forward = migration("0004_processing_exhausted")
|
||||
expect(await runMongoMigrations(db, [...history, forward])).toEqual({
|
||||
applied: [forward.name],
|
||||
skipped: history.map(({ name }) => name),
|
||||
})
|
||||
expect(await rows.findOne({ _id: row.id })).toEqual(before)
|
||||
expect(await rows.indexes()).toEqual(indexes)
|
||||
const [collectionAfter] = await db.listCollections({ name: "documents_document" }).toArray()
|
||||
if (!collectionBefore || !("info" in collectionBefore) || !collectionAfter || !("info" in collectionAfter))
|
||||
throw new Error("missing collection metadata")
|
||||
expect(collectionBefore.info?.uuid).toBeDefined()
|
||||
expect(collectionAfter.info?.uuid).toEqual(collectionBefore.info?.uuid)
|
||||
const failed = await repo.update(row.id, terminal, row.version)
|
||||
expect(await repo.get(row.id)).toEqual(failed)
|
||||
expect(failed).toMatchObject({ ...terminal, filename: input.filename, fileId: input.fileId })
|
||||
expect(await rows.countDocuments()).toBe(1)
|
||||
// All current codes and null remain usable; no blanket weakening of the enum.
|
||||
for (const failureCode of [...FAILURE_CODES, null]) {
|
||||
await repo.insert({ ...input, fileId: newId(), status: "failed", failureCode })
|
||||
}
|
||||
await expect(rows.updateOne({ _id: row.id }, { $set: { failure_code: "unknown_failure" } })).rejects.toThrow(
|
||||
/[Dd]ocument failed validation/,
|
||||
)
|
||||
// Zod still rejects an unknown code before an adapter write too.
|
||||
await expect(repo.update(row.id, { failureCode: "unknown_failure" as never }, failed.version)).rejects.toThrow()
|
||||
for (const scope of [
|
||||
makeTenant().project,
|
||||
{ ...tenant.project, workspaceId: tenant.siblingWorkspace.workspaceId },
|
||||
tenant.siblingProject,
|
||||
]) {
|
||||
const other = data.forScope(scope).of(documents)
|
||||
expect(await other.get(row.id)).toBeNull()
|
||||
await expect(other.update(row.id, { status: "processing" }, failed.version)).rejects.toThrow()
|
||||
}
|
||||
expect(await repo.get(row.id)).toEqual(failed)
|
||||
const stored = await rows.find().sort({ _id: 1 }).toArray()
|
||||
expect(await runMongoMigrations(db, [...history, forward])).toEqual({
|
||||
applied: [],
|
||||
skipped: [...history, forward].map(({ name }) => name),
|
||||
})
|
||||
expect(await rows.find().sort({ _id: 1 }).toArray()).toEqual(stored)
|
||||
} finally {
|
||||
await db.dropDatabase()
|
||||
await data.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,233 @@
|
||||
// What the extraction does with a page by its class (ocr-service OCR3.15): a drawing and an empty page
|
||||
// are never asked about and are reported; a scan and a table are sent as image and text; a text page as
|
||||
// text; the lines that repeat on most pages are not in the prompt; and a verified quote names the
|
||||
// reader's block. The model is a recording gateway: what it was sent is what is asserted.
|
||||
|
||||
import type { ChatMessage, GenerateObjectOptions, ModelGateway } from "@conusai-kit/platform"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { LOW_TRUST_NOTE, modelLineExtractor, PAGE_TEXT_CLOSE, PAGE_TEXT_OPEN } from "../adapters/model-extractor.js"
|
||||
import type { PageClass } from "../contract/index.js"
|
||||
import { planPage, promptTextOf } from "../core/page-plan.js"
|
||||
import type { ExtractionInput, PageImage, ReadDocument, ReadPage } from "../core/ports.js"
|
||||
import { noCache } from "../core/stage-cache.js"
|
||||
|
||||
/** A row long enough that a page of it is not weak for want of text (MIN_PAGE_TEXT_CHARS). */
|
||||
const SHEET_ROW = "Reference: QR-2026-0417 (rev 2)"
|
||||
|
||||
const png: PageImage = { bytes: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]), mediaType: "image/png" }
|
||||
|
||||
function page(index: number, pageClass: PageClass, rows: readonly string[], over: Partial<ReadPage> = {}): ReadPage {
|
||||
return {
|
||||
index,
|
||||
width: 1000,
|
||||
height: 1400,
|
||||
class: pageClass,
|
||||
source: pageClass === "text" ? "text_layer" : "ocr",
|
||||
text: rows.join("\n"),
|
||||
lines: rows.map((text, order) => ({
|
||||
text,
|
||||
bbox: [100, 100 + order * 40, 600, 130 + order * 40] as const,
|
||||
confidence: 0.97,
|
||||
order,
|
||||
repeated: false,
|
||||
})),
|
||||
blocks: rows.map((_, order) => ({
|
||||
id: `p${index + 1}-b${order + 1}`,
|
||||
bbox: [90, 90 + order * 40, 700, 140 + order * 40] as const,
|
||||
order,
|
||||
lines: [order],
|
||||
})),
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
const documentOf = (...pages: ReadPage[]): ReadDocument => ({
|
||||
engine: { name: "fixture", version: "1" },
|
||||
page_count: pages.length,
|
||||
pages,
|
||||
})
|
||||
|
||||
interface Sent {
|
||||
messages: ChatMessage[]
|
||||
options: GenerateObjectOptions
|
||||
}
|
||||
|
||||
/** A gateway that records every request and answers with the lines it is given for the page it sees. */
|
||||
function recording(answer: (text: string) => Array<{ label: string; value: string; quote: string }>) {
|
||||
const sent: Sent[] = []
|
||||
const gateway: ModelGateway = {
|
||||
async generateObject(_schema, messages, options) {
|
||||
sent.push({ messages: [...messages], options: options ?? {} })
|
||||
const user = messages.find((message) => message.role === "user")
|
||||
const parts = Array.isArray(user?.content) ? user.content : []
|
||||
const text = parts.map((part) => ("text" in part ? part.text : "")).join("\n")
|
||||
const from = text.indexOf(PAGE_TEXT_OPEN) + PAGE_TEXT_OPEN.length + 1
|
||||
const to = text.lastIndexOf(PAGE_TEXT_CLOSE) - 1
|
||||
return { value: { lines: answer(text.slice(from, to)) }, model: "m" } as never
|
||||
},
|
||||
}
|
||||
return { gateway, sent }
|
||||
}
|
||||
|
||||
const partsOf = (sent: Sent): Array<{ type: string; text?: string }> => {
|
||||
const user = sent.messages.find((message) => message.role === "user")
|
||||
return (Array.isArray(user?.content) ? user.content : []) as Array<{ type: string; text?: string }>
|
||||
}
|
||||
|
||||
const input = (document: ReadDocument, pageImage: ExtractionInput["pageImage"]): ExtractionInput => ({
|
||||
document,
|
||||
pageImage,
|
||||
cache: noCache,
|
||||
})
|
||||
|
||||
const reference = (text: string) =>
|
||||
text
|
||||
.split("\n")
|
||||
.filter((row) => row.includes(":"))
|
||||
.map((row) => ({ label: row.split(":")[0] as string, value: (row.split(":")[1] as string).trim(), quote: row }))
|
||||
|
||||
describe("the plan for a page", () => {
|
||||
const by = (pageClass: PageClass, over: Partial<ReadPage> = {}) =>
|
||||
planPage(page(0, pageClass, [SHEET_ROW], over), ["drawing", "empty"])
|
||||
|
||||
it("leaves out the classes it is told to, whatever else is true of the page", () => {
|
||||
expect(by("drawing")).toEqual({ kind: "skip" })
|
||||
expect(by("empty")).toEqual({ kind: "skip" })
|
||||
expect(planPage(page(0, "scan", [SHEET_ROW]), ["scan"])).toEqual({ kind: "skip" })
|
||||
})
|
||||
|
||||
it("shows a scan its image and says its text is a recognition; a table gets image and text", () => {
|
||||
expect(by("scan")).toEqual({ kind: "read", image: true, lowTrustText: true })
|
||||
expect(by("table")).toMatchObject({ kind: "read", image: true })
|
||||
})
|
||||
|
||||
it("reads a text page as text, with the image only when the reader was unsure of it", () => {
|
||||
expect(by("text")).toEqual({ kind: "read", image: false, lowTrustText: false })
|
||||
const unsure = page(0, "text", [SHEET_ROW])
|
||||
const weak = { ...unsure, lines: unsure.lines.map((line) => ({ ...line, confidence: 0.4 })) }
|
||||
expect(planPage(weak, ["drawing", "empty"])).toMatchObject({ kind: "read", image: true })
|
||||
})
|
||||
|
||||
it("reads a drawing when the deployment lets it, with its image", () => {
|
||||
expect(planPage(page(0, "drawing", ["A-101"]), [])).toMatchObject({ kind: "read", image: true })
|
||||
})
|
||||
|
||||
it("drops the lines flagged as repeating from the text the model is shown", () => {
|
||||
const rows = ["ACME Specification", "Reference: QR-1", "Page footer"]
|
||||
const shown = page(0, "text", rows)
|
||||
const lines = shown.lines.map((line, at) => ({ ...line, repeated: at !== 1 }))
|
||||
expect(promptTextOf({ ...shown, lines })).toBe("Reference: QR-1")
|
||||
expect(promptTextOf({ ...shown, lines: [] })).toBe(rows.join("\n"))
|
||||
})
|
||||
})
|
||||
|
||||
describe("the extractor and the page classes", () => {
|
||||
it("asks nothing about a drawing or an empty page and reports them as skipped, in page order", async () => {
|
||||
const { gateway, sent } = recording(reference)
|
||||
const document = documentOf(
|
||||
page(0, "text", ["Reference: QR-1"]),
|
||||
page(
|
||||
1,
|
||||
"drawing",
|
||||
Array.from({ length: 400 }, (_, at) => `A-${at} 2400`),
|
||||
),
|
||||
page(2, "empty", []),
|
||||
)
|
||||
const result = await modelLineExtractor({ gateway, model: "m" }).extract(input(document, async () => png))
|
||||
expect(sent).toHaveLength(1)
|
||||
expect(result.lines.map((line) => [line.page, line.value])).toEqual([[1, "QR-1"]])
|
||||
expect(result.skipped).toEqual([
|
||||
{ page: 2, pageClass: "drawing" },
|
||||
{ page: 3, pageClass: "empty" },
|
||||
])
|
||||
})
|
||||
|
||||
it("lets a deployment read drawings: the list of excluded classes is the option's", async () => {
|
||||
const { gateway, sent } = recording(reference)
|
||||
const document = documentOf(page(0, "drawing", ["Scale: 1:50"]))
|
||||
const result = await modelLineExtractor({ gateway, model: "m", excludedPageClasses: [] }).extract(
|
||||
input(document, async () => png),
|
||||
)
|
||||
expect(sent).toHaveLength(1)
|
||||
expect(result.skipped).toEqual([])
|
||||
})
|
||||
|
||||
it("sends a scan as text and image, the text marked low trust; a text page as text alone", async () => {
|
||||
const { gateway, sent } = recording(reference)
|
||||
const asked: number[] = []
|
||||
const document = documentOf(page(0, "scan", ["Reference: QR-1"]), page(1, "text", [SHEET_ROW]))
|
||||
await modelLineExtractor({ gateway, model: "m" }).extract(
|
||||
input(document, async (index) => {
|
||||
asked.push(index)
|
||||
return png
|
||||
}),
|
||||
)
|
||||
expect(asked).toEqual([0])
|
||||
const kinds = sent.map((one) => partsOf(one).map((part) => part.type))
|
||||
expect(kinds).toContainEqual(["text", "image_url"])
|
||||
expect(kinds).toContainEqual(["text"])
|
||||
const scanText = partsOf(sent.find((one) => partsOf(one).length === 2) as Sent)[0]?.text
|
||||
expect(scanText).toContain(LOW_TRUST_NOTE)
|
||||
const textText = partsOf(sent.find((one) => partsOf(one).length === 1) as Sent)[0]?.text
|
||||
expect(textText).not.toContain(LOW_TRUST_NOTE)
|
||||
})
|
||||
|
||||
it("leaves a header that repeats on most pages out of the prompt, and still verifies against the page", async () => {
|
||||
const { gateway, sent } = recording(reference)
|
||||
const rows = ["ACME: Specification", "Reference: QR-1"]
|
||||
const withHeader = page(0, "text", rows)
|
||||
const document = documentOf({
|
||||
...withHeader,
|
||||
lines: withHeader.lines.map((line, at) => ({ ...line, repeated: at === 0 })),
|
||||
})
|
||||
const result = await modelLineExtractor({ gateway, model: "m" }).extract(input(document, async () => undefined))
|
||||
const text = partsOf(sent[0] as Sent)[0]?.text ?? ""
|
||||
expect(text).toContain("Reference: QR-1")
|
||||
expect(text).not.toContain("ACME")
|
||||
expect(result.lines.map((line) => [line.label, line.verification])).toEqual([["Reference", "verified"]])
|
||||
})
|
||||
|
||||
it("gives a verified line the block it sits in, and that block's box in place of the lines'", async () => {
|
||||
const { gateway } = recording(reference)
|
||||
const document = documentOf(page(0, "text", ["Reference: QR-1", "Issued: 2026-04-01"]))
|
||||
const { lines } = await modelLineExtractor({ gateway, model: "m" }).extract(input(document, async () => undefined))
|
||||
const second = lines.find((line) => line.label === "Issued")
|
||||
expect(second?.blockId).toBe("p1-b2")
|
||||
// block p1-b2 is [90, 130, 700, 180] on a 1000 x 1400 page, not the line's [100, 140, 600, 170]
|
||||
expect(second?.box).toEqual({ x: 0.09, y: 130 / 1400, width: 0.61, height: 50 / 1400 })
|
||||
})
|
||||
|
||||
it("marks a quote the model read off the image, and the text does not hold, as a scanned quote to check", async () => {
|
||||
const { gateway } = recording(() => [{ label: "Qty", value: "12", quote: "Qty 12 pcs" }])
|
||||
const document = documentOf(page(0, "scan", ["Delivery note"]), page(1, "text", ["Delivery note"]))
|
||||
const { lines } = await modelLineExtractor({ gateway, model: "m" }).extract(
|
||||
input(document, async (index) => (index === 0 ? png : undefined)),
|
||||
)
|
||||
const [scan, text] = lines
|
||||
expect(scan).toMatchObject({ page: 1, verification: "needs_review", note: "scanned_quote", confidence: 0 })
|
||||
// the same unfound quote on a page the model saw no image of carries no such note
|
||||
expect(text).toMatchObject({ page: 2, verification: "needs_review" })
|
||||
expect(text?.note).toBeUndefined()
|
||||
})
|
||||
|
||||
it("keys the cache by the image: another image of the same page is asked again", async () => {
|
||||
const stored = new Map<string, unknown>()
|
||||
const cache = {
|
||||
async get(stage: string, key: string) {
|
||||
return stored.get(`${stage}/${key}`) as never
|
||||
},
|
||||
async put(stage: string, key: string, value: unknown) {
|
||||
stored.set(`${stage}/${key}`, value)
|
||||
},
|
||||
}
|
||||
const { gateway, sent } = recording(reference)
|
||||
const document = documentOf(page(0, "scan", ["Reference: QR-1"]))
|
||||
const extractor = modelLineExtractor({ gateway, model: "m" })
|
||||
const run = (image: PageImage) => extractor.extract({ document, pageImage: async () => image, cache })
|
||||
await run(png)
|
||||
await run(png)
|
||||
expect(sent).toHaveLength(1)
|
||||
await run({ ...png, bytes: new Uint8Array([0x89, 0x50, 0x4e, 0x47, 9, 9, 9]) })
|
||||
expect(sent).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,211 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
import type { DocumentItems, Item, ItemCitation } from "@conusai-kit/platform"
|
||||
import { describe, expect, it, vi } from "vitest"
|
||||
import { boxOf, candidateOf, labelAndValueOf, platformItemsExtractor } from "../adapters/platform-items-extractor.js"
|
||||
import { feedbackOf, itemRefOf } from "../core/item-feedback.js"
|
||||
import { ExtractionFailed, type ReadDocument, type ReadPage } from "../core/ports.js"
|
||||
import { noCache } from "../core/stage-cache.js"
|
||||
|
||||
const SHA = "a".repeat(64)
|
||||
|
||||
const cited = (page: number): ItemCitation => ({
|
||||
kind: "pdf_page",
|
||||
file_sha256: SHA,
|
||||
page,
|
||||
lines: [1],
|
||||
bbox: [100, 400, 900, 500],
|
||||
quote: "1 Kabelis Al 4x25 m 120",
|
||||
})
|
||||
|
||||
const item = (overrides: Partial<Item> = {}): Item => ({
|
||||
id: "p1-l1",
|
||||
section: { title: "Medžiagos", class: "material", page: 1 },
|
||||
position: "1",
|
||||
name: "Kabelis Al 4x25",
|
||||
class: "material",
|
||||
unit: { text: "m", code: "MTR" },
|
||||
quantity: { value: "120", text: "120", column: "Kiekis", refusal: null },
|
||||
spec: null,
|
||||
citations: [cited(1)],
|
||||
checks: [{ check: "quote_verbatim", status: "passed" }],
|
||||
agreement: { status: "auto_accepted", reader: true, model: true },
|
||||
method: "reader_and_model",
|
||||
confidence: 0.98,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const page = (index: number, text: string, cls: ReadPage["class"] = "table"): ReadPage => ({
|
||||
index,
|
||||
width: 1000,
|
||||
height: 2000,
|
||||
class: cls,
|
||||
text,
|
||||
lines: [],
|
||||
blocks: [],
|
||||
})
|
||||
const first = page(0, "Medžiagos\n1 Kabelis Al 4x25 m 120")
|
||||
const result: Pick<DocumentItems, "run_id" | "models"> = {
|
||||
run_id: 31,
|
||||
models: [{ stage: "rows", model: "claude-haiku-5-5" }],
|
||||
}
|
||||
const pageOf = (n: number) => (n === 1 ? first : undefined)
|
||||
|
||||
describe("platform items as lines", () => {
|
||||
it("maps an item to a verified line, its box as fractions of the page read", () => {
|
||||
expect(candidateOf(item(), result, pageOf)).toEqual({
|
||||
label: "Kabelis Al 4x25",
|
||||
value: "120 m",
|
||||
page: 1,
|
||||
quote: "1 Kabelis Al 4x25 m 120",
|
||||
box: { x: 0.1, y: 0.2, width: 0.8, height: expect.closeTo(0.05) },
|
||||
confidence: 0.98,
|
||||
model: "claude-haiku-5-5",
|
||||
verification: "verified",
|
||||
itemRef: "31:p1-l1",
|
||||
})
|
||||
})
|
||||
|
||||
it("asks for review when the platform does, when a check failed, or when the quote is not on the page", () => {
|
||||
expect(
|
||||
candidateOf(item({ agreement: { status: "review_required", reader: true, model: false } }), result, pageOf)
|
||||
?.verification,
|
||||
).toBe("needs_review")
|
||||
expect(
|
||||
candidateOf(item({ checks: [{ check: "unit_known", status: "failed" }] }), result, pageOf)?.verification,
|
||||
).toBe("needs_review")
|
||||
expect(candidateOf(item(), result, () => page(0, "another page"))?.verification).toBe("needs_review")
|
||||
})
|
||||
|
||||
it("proposes nothing for an item without a citation, and no reference without a run", () => {
|
||||
expect(candidateOf(item({ citations: [] }), result, pageOf)).toBeUndefined()
|
||||
expect(candidateOf(item(), { ...result, run_id: null }, pageOf)?.itemRef).toBeUndefined()
|
||||
})
|
||||
|
||||
it("cites only a PDF page: an item cited by a sheet's cells alone is not proposed", () => {
|
||||
const sheet: ItemCitation = {
|
||||
kind: "sheet",
|
||||
file_sha256: SHA,
|
||||
sheet: "BOQ",
|
||||
cell_range: "A3:F3",
|
||||
quote: "1 Kabelis",
|
||||
}
|
||||
expect(candidateOf(item({ citations: [sheet] }), result, pageOf)).toBeUndefined()
|
||||
expect(candidateOf(item({ citations: [sheet, cited(1)] }), result, pageOf)).toMatchObject({ page: 1 })
|
||||
})
|
||||
|
||||
it("keeps no box it cannot place: no geometry, no page size, or an empty box", () => {
|
||||
expect(boxOf(null, first)).toBeUndefined()
|
||||
expect(boxOf([0, 0, 10, 10], undefined)).toBeUndefined()
|
||||
expect(boxOf([10, 10, 10, 20], first)).toBeUndefined()
|
||||
expect(boxOf([-5, 0, 2000, 2000], first)).toEqual({ x: 0, y: 0, width: 1, height: 1 })
|
||||
})
|
||||
|
||||
it("reads a label/value item with no unit as its quantity text", () => {
|
||||
expect(
|
||||
labelAndValueOf(
|
||||
item({
|
||||
name: "Total",
|
||||
unit: { text: "", code: null },
|
||||
quantity: { value: null, text: "EUR 5", column: "", refusal: "not_a_number" },
|
||||
}),
|
||||
),
|
||||
).toEqual({
|
||||
label: "Total",
|
||||
value: "EUR 5",
|
||||
})
|
||||
})
|
||||
|
||||
it("runs the profile over the job that read the document, once per job, and leaves drawings out", async () => {
|
||||
const submit = vi.fn(async () => ({ id: 31, jobId: 4, task: "items" as const, status: "queued" as const }) as never)
|
||||
const wait = vi.fn(async () => ({
|
||||
schema_version: "conusai.documents.items/v1" as const,
|
||||
job_id: 4,
|
||||
run_id: 31,
|
||||
file_sha256: SHA,
|
||||
profile: { id: "bill_of_quantities", version: 1 },
|
||||
pipeline_revision: "x",
|
||||
models: result.models,
|
||||
pages: [],
|
||||
items: [item(), item({ id: "p2-l1", citations: [cited(2)] })],
|
||||
numbering: [],
|
||||
usage: {
|
||||
pages_read: 2,
|
||||
pages_sent_to_model: 1,
|
||||
input_tokens: 1,
|
||||
output_tokens: 1,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
wall_ms: 1,
|
||||
api_ms: 1,
|
||||
cost_usd: "0",
|
||||
},
|
||||
warnings: [],
|
||||
}))
|
||||
const document: ReadDocument = {
|
||||
engine: { name: "t", version: "1" },
|
||||
page_count: 2,
|
||||
pages: [first, page(1, "1 Kabelis Al 4x25 m 120", "drawing")],
|
||||
}
|
||||
const extractor = platformItemsExtractor({
|
||||
items: { submit, wait },
|
||||
profile: "bill_of_quantities",
|
||||
keep: ["material"],
|
||||
})
|
||||
const extraction = await extractor.extract({
|
||||
document,
|
||||
ocrJobId: 4,
|
||||
pageImage: async () => undefined,
|
||||
cache: noCache,
|
||||
})
|
||||
|
||||
expect(submit).toHaveBeenCalledWith(4, {
|
||||
profile: "bill_of_quantities",
|
||||
keep: ["material"],
|
||||
idempotencyKey: "documents.items.4.bill_of_quantities",
|
||||
})
|
||||
expect(extraction.lines.map((line) => line.page)).toEqual([1])
|
||||
expect(extraction.skipped).toEqual([{ page: 2, pageClass: "drawing" }])
|
||||
expect(extractor.version).toBe("items.bill_of_quantities.v1")
|
||||
await expect(
|
||||
extractor.extract({ document, pageImage: async () => undefined, cache: noCache }),
|
||||
).rejects.toBeInstanceOf(ExtractionFailed)
|
||||
})
|
||||
})
|
||||
|
||||
describe("a decision as item feedback", () => {
|
||||
const change = {
|
||||
lineKey: "f".repeat(64),
|
||||
label: "Kabelis Al 4x25",
|
||||
value: "120 m",
|
||||
page: 1,
|
||||
position: 0,
|
||||
quote: "1 Kabelis Al 4x25 m 120",
|
||||
confidence: 0.98,
|
||||
verification: "verified",
|
||||
edited: false,
|
||||
source: {
|
||||
fileSha256: SHA,
|
||||
ocrJobId: 4,
|
||||
ocrEngine: { name: "t", version: "1" },
|
||||
model: "claude-haiku-5-5",
|
||||
promptVersion: "items.bill_of_quantities.v1",
|
||||
itemRef: "31:p1-l1",
|
||||
},
|
||||
}
|
||||
|
||||
it("names the run and the item, and sends the reviewer's words only for a correction", () => {
|
||||
expect(itemRefOf(change)).toEqual({ runId: 31, itemId: "p1-l1" })
|
||||
expect(feedbackOf("approved", change)).toEqual({ runId: 31, feedback: { item_id: "p1-l1", decision: "approved" } })
|
||||
expect(feedbackOf("rejected", change)).toEqual({ runId: 31, feedback: { item_id: "p1-l1", decision: "rejected" } })
|
||||
expect(feedbackOf("approved", { ...change, edited: true, value: "125 m" })).toEqual({
|
||||
runId: 31,
|
||||
feedback: { item_id: "p1-l1", decision: "edited", fields: { label: "Kabelis Al 4x25", value: "125 m" } },
|
||||
})
|
||||
})
|
||||
|
||||
it("is nothing for a line the platform did not extract", () => {
|
||||
const { itemRef: _, ...source } = change.source
|
||||
expect(feedbackOf("approved", { ...change, source })).toBeUndefined()
|
||||
expect(feedbackOf("approved", { not: "a line" })).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,45 @@
|
||||
// Reads a ZIP the way a spreadsheet application does: from the central directory at the end. It is
|
||||
// the test's independent reader of what `buildWorkbook` wrote, so the writer is not checked by itself.
|
||||
|
||||
import { inflateRawSync } from "node:zlib"
|
||||
|
||||
export function unzip(bytes: Uint8Array): Map<string, string> {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||
let eocd = bytes.length - 22
|
||||
while (eocd >= 0 && view.getUint32(eocd, true) !== 0x06054b50) eocd -= 1
|
||||
if (eocd < 0) throw new Error("no end-of-central-directory record")
|
||||
const entries = view.getUint16(eocd + 10, true)
|
||||
let at = view.getUint32(eocd + 16, true)
|
||||
const files = new Map<string, string>()
|
||||
for (let i = 0; i < entries; i += 1) {
|
||||
if (view.getUint32(at, true) !== 0x02014b50) throw new Error("a bad central directory entry")
|
||||
const method = view.getUint16(at + 10, true)
|
||||
const size = view.getUint32(at + 20, true)
|
||||
const nameLength = view.getUint16(at + 28, true)
|
||||
const extraLength = view.getUint16(at + 30, true)
|
||||
const commentLength = view.getUint16(at + 32, true)
|
||||
const local = view.getUint32(at + 42, true)
|
||||
const name = new TextDecoder().decode(bytes.subarray(at + 46, at + 46 + nameLength))
|
||||
const dataAt = local + 30 + view.getUint16(local + 26, true) + view.getUint16(local + 28, true)
|
||||
const raw = bytes.subarray(dataAt, dataAt + size)
|
||||
files.set(name, new TextDecoder().decode(method === 8 ? inflateRawSync(raw) : raw))
|
||||
at += 46 + nameLength + extraLength + commentLength
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
/** The text and number cells of the first sheet, row by row. */
|
||||
export function sheetRows(bytes: Uint8Array): Array<Array<string | number>> {
|
||||
const sheet = unzip(bytes).get("xl/worksheets/sheet1.xml") ?? ""
|
||||
return [...sheet.matchAll(/<row [^>]*>(.*?)<\/row>/gs)].map(([, row]) =>
|
||||
[...(row as string).matchAll(/<c ([^>]*)>(.*?)<\/c>/gs)].map(([, attributes, body]) => {
|
||||
if (!(attributes as string).includes('t="inlineStr"')) return Number(/<v>(.*?)<\/v>/s.exec(body as string)?.[1])
|
||||
const text = /<t[^>]*>(.*?)<\/t>/s.exec(body as string)?.[1] ?? ""
|
||||
return text
|
||||
.replace(/"/g, '"')
|
||||
.replace(/>/g, ">")
|
||||
.replace(/</g, "<")
|
||||
.replace(/&/g, "&")
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// The components' markup, rendered on the server the way a page renders it: what a screen reader and
|
||||
// a keyboard user get is in the HTML, so the accessibility basics are asserted there -- a name for
|
||||
// every control, a caption and scoped headers for every table, a status that says what a colour would.
|
||||
|
||||
import { buildRegistry } from "@conusai-kit/core"
|
||||
import { builtInCatalogues, createI18n, uiLabels } from "@conusai-kit/i18n"
|
||||
import { createDataClient, DataProvider } from "@conusai-kit/ui/data"
|
||||
import { UiLabelsProvider } from "@conusai-kit/ui/primitives"
|
||||
import { renderToStaticMarkup as renderMarkup } from "react-dom/server"
|
||||
import { describe, expect, it, vi } from "vitest"
|
||||
import { appModules } from "../../index.js"
|
||||
import type { DocumentView, LineView } from "../contract/index.js"
|
||||
import {
|
||||
DocumentList,
|
||||
documentHeader,
|
||||
ExportButton,
|
||||
exportLabels,
|
||||
headerLabels,
|
||||
LineReview,
|
||||
listLabels,
|
||||
reviewLabels,
|
||||
UploadForm,
|
||||
uploadLabels,
|
||||
} from "../ui/index.js"
|
||||
|
||||
const en = buildRegistry([...appModules]).catalogues.en?.documents ?? {}
|
||||
const t = (key: string, values?: Record<string, string | number>) => {
|
||||
let text = en[key as keyof typeof en] ?? `?${key}`
|
||||
for (const [name, value] of Object.entries(values ?? {})) text = text.replace(`{${name}}`, String(value))
|
||||
return text
|
||||
}
|
||||
const slugs = { org: "acme", workspace: "ws", project: "p" }
|
||||
const shared = { failed: "That did not work: {detail}", aiDisclosure: "AI-generated content", pageWord: "Page" }
|
||||
|
||||
const line = (over: Partial<LineView> = {}): LineView => ({
|
||||
lineId: "l1",
|
||||
expectedVersion: 1,
|
||||
status: "pending",
|
||||
label: "Reference",
|
||||
value: "QR-1",
|
||||
page: 1,
|
||||
quote: "Reference: QR-1",
|
||||
box: { x: 0.1, y: 0.2, width: 0.5, height: 0.03 },
|
||||
confidence: 0.97,
|
||||
edited: false,
|
||||
verification: "verified",
|
||||
note: null,
|
||||
blockId: null,
|
||||
model: "fixture+gateway",
|
||||
promptVersion: "extract-lines.v2",
|
||||
decidedAt: null,
|
||||
...over,
|
||||
})
|
||||
|
||||
const kitStrings = uiLabels(createI18n("en", builtInCatalogues))
|
||||
const renderToStaticMarkup = (node: React.ReactNode) =>
|
||||
renderMarkup(<UiLabelsProvider labels={kitStrings}>{node}</UiLabelsProvider>)
|
||||
|
||||
const inProvider = (node: React.ReactNode) =>
|
||||
renderToStaticMarkup(
|
||||
<DataProvider client={createDataClient({ fetch: async () => new Response(null, { status: 501 }) })}>
|
||||
{node}
|
||||
</DataProvider>,
|
||||
)
|
||||
|
||||
const review = (lines: LineView[]) =>
|
||||
inProvider(
|
||||
<LineReview
|
||||
slugs={slugs}
|
||||
fileId="f1"
|
||||
lines={lines}
|
||||
labels={reviewLabels(t, shared)}
|
||||
locale="en"
|
||||
onDecided={async () => undefined}
|
||||
/>,
|
||||
)
|
||||
|
||||
describe("the review table", () => {
|
||||
const html = review([
|
||||
line(),
|
||||
line({ lineId: "l2", label: "Total", status: "approved" }),
|
||||
line({ lineId: "l3", label: "Date", status: "rejected" }),
|
||||
])
|
||||
|
||||
it("tells a reader the content is machine-proposed, in text", () => {
|
||||
expect(html).toContain("AI-generated content")
|
||||
expect(html).toContain("Proposed by fixture+gateway extract-lines.v2")
|
||||
})
|
||||
|
||||
it("is a table with a caption and a scoped header for every column", () => {
|
||||
expect(html).toMatch(/<caption\b[^>]*>Extracted lines<\/caption>/)
|
||||
expect(html.match(/<th\b[^>]*scope="col"[^>]*>/g)).toHaveLength(7)
|
||||
expect(html).toMatch(/<th\b[^>]*scope="row"[^>]*>Reference<\/th>/)
|
||||
})
|
||||
|
||||
it("names each action by the line it acts on, so a list of buttons is not a list of 'Approve'", () => {
|
||||
for (const name of ["Approve Reference", "Edit Reference", "Reject Reference"]) expect(html).toContain(name)
|
||||
})
|
||||
|
||||
it("offers no action on a line that has been decided, and says what was decided in words", () => {
|
||||
expect(html).not.toContain("Approve Total")
|
||||
expect(html).not.toContain("Reject Date")
|
||||
for (const word of ["To review", "Approved", "Rejected"]) expect(html).toContain(word)
|
||||
})
|
||||
|
||||
it("shows the source of every line, labelled with its page", () => {
|
||||
expect(html).toContain('aria-label="Source, page 1"')
|
||||
expect(html).toContain("Reference: QR-1")
|
||||
})
|
||||
|
||||
it("approves every verified pending line at once, only while there is one", () => {
|
||||
expect(html).toContain("Approve all verified lines")
|
||||
expect(review([line({ status: "approved" })])).not.toContain("Approve all verified lines")
|
||||
})
|
||||
|
||||
it("marks a line whose quote was not found, and does not offer it to the bulk approval", () => {
|
||||
const flagged = review([line({ verification: "needs_review" })])
|
||||
expect(flagged).toContain("Needs review: the quoted words were not found on the page.")
|
||||
expect(flagged).not.toContain("Approve all verified lines")
|
||||
expect(html).not.toContain("Needs review")
|
||||
})
|
||||
|
||||
it("says a quote read off a scanned page must be checked against the image, instead of the plain note", () => {
|
||||
const flagged = review([line({ verification: "needs_review", note: "scanned_quote" })])
|
||||
expect(flagged).toContain("comes from a scanned page")
|
||||
expect(flagged).toContain("Check it against the page image.")
|
||||
expect(flagged).not.toContain("were not found on the page")
|
||||
})
|
||||
|
||||
it("names the reader's block beside the source when it gave one", () => {
|
||||
expect(review([line({ blockId: "p3-b2" })])).toContain("Block p3-b2")
|
||||
expect(review([line()])).not.toContain("Block ")
|
||||
})
|
||||
|
||||
it("says so when there is nothing to review", () => {
|
||||
expect(review([])).toContain("No lines were proposed for this document.")
|
||||
})
|
||||
})
|
||||
|
||||
describe("the document list", () => {
|
||||
const documents: DocumentView[] = [
|
||||
{
|
||||
documentId: "d1",
|
||||
fileId: "f1",
|
||||
filename: "report.pdf",
|
||||
status: "extracted",
|
||||
failureCode: null,
|
||||
pageCount: 2,
|
||||
skippedPages: [],
|
||||
pending: 3,
|
||||
approved: 1,
|
||||
uploadedAt: "2026-10-19T09:00:00.000Z",
|
||||
},
|
||||
{
|
||||
documentId: "d2",
|
||||
fileId: "f2",
|
||||
filename: "scan.png",
|
||||
status: "failed",
|
||||
failureCode: "ocr_failed",
|
||||
pageCount: null,
|
||||
skippedPages: [],
|
||||
pending: 0,
|
||||
approved: 0,
|
||||
uploadedAt: "2026-10-19T09:05:00.000Z",
|
||||
},
|
||||
{
|
||||
documentId: null,
|
||||
fileId: "f3",
|
||||
filename: "new.pdf",
|
||||
status: "queued",
|
||||
failureCode: null,
|
||||
pageCount: null,
|
||||
skippedPages: [],
|
||||
pending: 0,
|
||||
approved: 0,
|
||||
uploadedAt: "2026-10-19T09:06:00.000Z",
|
||||
},
|
||||
]
|
||||
const html = renderToStaticMarkup(
|
||||
<DocumentList
|
||||
documents={documents}
|
||||
hrefFor={(id) => `/d/${id}`}
|
||||
downloadHrefFor={(id) => `/api/v1/files.file.download?fileId=${id}`}
|
||||
labels={listLabels(t)}
|
||||
formatDate={(iso) => iso.slice(0, 10)}
|
||||
/>,
|
||||
)
|
||||
|
||||
it("links each document by a name that says which one", () => {
|
||||
expect(html).toContain('href="/d/f1"')
|
||||
expect(html).toContain('aria-label="Review report.pdf"')
|
||||
expect(html).toContain('href="/api/v1/files.file.download?fileId=f1"')
|
||||
expect(html).toContain(">Download report.pdf</a>")
|
||||
})
|
||||
|
||||
it("writes each state and each failure as text", () => {
|
||||
expect(html).toContain("Ready for review")
|
||||
expect(html).toContain("Could not be read The reader could not process this file.")
|
||||
expect(html).toContain("Waiting to be read")
|
||||
})
|
||||
|
||||
it("is a captioned table, or a sentence when there is nothing to list", () => {
|
||||
expect(html).toContain('aria-label="Documents"')
|
||||
expect(
|
||||
renderToStaticMarkup(
|
||||
<DocumentList
|
||||
documents={[]}
|
||||
hrefFor={() => ""}
|
||||
downloadHrefFor={() => ""}
|
||||
labels={listLabels(t)}
|
||||
formatDate={() => ""}
|
||||
/>,
|
||||
),
|
||||
).toContain("No documents yet.")
|
||||
})
|
||||
})
|
||||
|
||||
describe("the export button", () => {
|
||||
const render = (approved: number) =>
|
||||
inProvider(<ExportButton slugs={slugs} fileId="f1" approved={approved} labels={exportLabels(t, shared.failed)} />)
|
||||
|
||||
it("is disabled with nothing approved, and the reason is the text its description points at", () => {
|
||||
const html = render(0)
|
||||
expect(html).toMatch(/<button[^>]*disabled=""[^>]*aria-describedby="([^"]+)"/)
|
||||
const id = /aria-describedby="([^"]+)"/.exec(html)?.[1] ?? ""
|
||||
expect(html).toContain(`id="${id}">Approve at least one line to export.`)
|
||||
})
|
||||
|
||||
it("waits for hydration even with approved lines, and explains what will be exported", () => {
|
||||
const html = render(2)
|
||||
expect(html).toContain('disabled=""')
|
||||
expect(html).toContain("Only approved lines are exported.")
|
||||
})
|
||||
})
|
||||
|
||||
describe("the document's header", () => {
|
||||
// Built wherever the live document renders, so the counts and the skipped pages follow a change.
|
||||
const view = (over: Partial<DocumentView> = {}): DocumentView => ({
|
||||
documentId: "d1",
|
||||
fileId: "f1",
|
||||
filename: "report.pdf",
|
||||
status: "extracted",
|
||||
failureCode: null,
|
||||
pageCount: 3,
|
||||
skippedPages: [],
|
||||
pending: 1,
|
||||
approved: 2,
|
||||
uploadedAt: "2026-10-19T09:00:00.000Z",
|
||||
...over,
|
||||
})
|
||||
|
||||
it("says the status and counts the lines with the language's plurals", () => {
|
||||
const header = documentHeader(view(), headerLabels(t), "en")
|
||||
expect(header.status).toBe("Status: Ready for review")
|
||||
expect(header.counts).toBe("1 line to review. 2 lines approved.")
|
||||
expect(header.waiting).toBeUndefined()
|
||||
expect(header.skipped).toBeUndefined()
|
||||
})
|
||||
|
||||
it("says a document nobody has read yet is waiting, and names a failure", () => {
|
||||
expect(documentHeader(view({ status: "processing" }), headerLabels(t), "en").waiting).toBeTruthy()
|
||||
const failed = documentHeader(view({ status: "failed", failureCode: "ocr_failed" }), headerLabels(t), "en")
|
||||
expect(failed.status).toMatch(/^Status: \S.* \S/)
|
||||
})
|
||||
|
||||
it("lists the pages the extraction left out, each with its kind", () => {
|
||||
const header = documentHeader(view({ skippedPages: [{ page: 3, pageClass: "drawing" }] }), headerLabels(t), "en")
|
||||
expect(header.skipped).toEqual({
|
||||
title: "Pages not read",
|
||||
pages: [{ page: 3, text: "Page 3: drawing. It was left out of the extraction." }],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("the upload form", () => {
|
||||
// The page's data client: the form uploads through the facade's `useUpload`, which asks the
|
||||
// client the app provides for a way to send a body.
|
||||
const html = renderToStaticMarkup(
|
||||
<DataProvider client={createDataClient({ fetch: async () => new Response(null, { status: 501 }) })}>
|
||||
<UploadForm
|
||||
slugs={slugs}
|
||||
accept={["application/pdf"]}
|
||||
labels={uploadLabels(t)}
|
||||
onUploaded={async () => undefined}
|
||||
/>
|
||||
</DataProvider>,
|
||||
)
|
||||
|
||||
it("labels its file input and describes it with the hint", () => {
|
||||
// React writes an input's attributes in its own order (`type` first), so the pattern does not fix one.
|
||||
const id = /<input [^>]*\bid="([^"]+)"[^>]*\bname="file"/.exec(html)?.[1]
|
||||
expect(id).toBeTruthy()
|
||||
expect(html).toMatch(new RegExp(`<label [^>]*for="${id}"[^>]*>Upload a document</label>`))
|
||||
expect(html).toContain("aria-describedby=")
|
||||
expect(html).toContain("PDF, PNG or JPEG, up to 50 MB.")
|
||||
})
|
||||
|
||||
it("asks the browser for the types the module reads and requires a file", () => {
|
||||
expect(html).toContain('accept="application/pdf"')
|
||||
expect(html).toContain('required=""')
|
||||
})
|
||||
})
|
||||
|
||||
// Explicit synthetic admission: module-local fixtures exercise the selected module.
|
||||
vi.mock("../../inclusion.json", () => ({
|
||||
default: { schemaVersion: 1, selected: ["documents"], everAdopted: ["documents"] },
|
||||
}))
|
||||
@@ -0,0 +1,59 @@
|
||||
import { readFileSync } from "node:fs"
|
||||
import { describe, expect, it } from "vitest"
|
||||
import type { ReadDocument } from "../core/ports.js"
|
||||
import { foldForComparison, isWeakPage, locateQuote, quoteIsOnPage } from "../core/verify.js"
|
||||
|
||||
const fixture = JSON.parse(
|
||||
readFileSync(new URL("./fixtures/ocr-document.json", import.meta.url), "utf8"),
|
||||
) as ReadDocument
|
||||
const [first, second] = fixture.pages as [ReadDocument["pages"][number], ReadDocument["pages"][number]]
|
||||
|
||||
describe("the comparison of a quote with a page", () => {
|
||||
it("folds whitespace, case and diacritics, and nothing else", () => {
|
||||
expect(foldForComparison(" Šarūnas \n\t ĄČĘĖĮŠŲŪŽ ")).toBe("sarunas aceeisuuz")
|
||||
// digits, punctuation and letters that are different letters stay different
|
||||
expect(foldForComparison("1 250,00")).not.toBe(foldForComparison("1 520,00"))
|
||||
expect(foldForComparison("a-b")).not.toBe(foldForComparison("a b"))
|
||||
expect(foldForComparison("ß")).toBe("ß")
|
||||
})
|
||||
|
||||
it("finds a quote that differs from the page only in what is folded", () => {
|
||||
expect(quoteIsOnPage(first.text, "reference: qr-2026-04")).toBe(true)
|
||||
expect(quoteIsOnPage(first.text, "Issued on: 2026-04-01\nThis line has no label")).toBe(true)
|
||||
})
|
||||
|
||||
it("does not find a quote that is not there, an empty one, or one from another page", () => {
|
||||
expect(quoteIsOnPage(first.text, "Reference: QR-2026-05")).toBe(false)
|
||||
expect(quoteIsOnPage(first.text, " ")).toBe(false)
|
||||
expect(quoteIsOnPage(first.text, "Total amount: 1 250,00 EUR")).toBe(false)
|
||||
expect(quoteIsOnPage(second.text, "Total amount: 1 250,00 EUR")).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe("where a quote sits", () => {
|
||||
it("is the box of the line, as fractions of the page, with that line's confidence", () => {
|
||||
const at = locateQuote(first, "REFERENCE: QR-2026-04")
|
||||
expect(at?.box).toEqual({ x: 0.1, y: 200 / 1400, width: 0.5, height: 40 / 1400 })
|
||||
expect(at?.confidence).toBe(0.97)
|
||||
})
|
||||
|
||||
it("spans the lines a quote runs over, and averages their confidence", () => {
|
||||
const at = locateQuote(first, "Reference: QR-2026-04\nIssued on: 2026-04-01")
|
||||
expect(at?.box).toEqual({ x: 0.1, y: 200 / 1400, width: 0.5, height: 100 / 1400 })
|
||||
expect(at?.confidence).toBeCloseTo(0.96)
|
||||
})
|
||||
|
||||
it("is nothing for a quote that is not on the lines, and no box for a page with no size", () => {
|
||||
expect(locateQuote(first, "Reference: QR-9")).toBeUndefined()
|
||||
expect(locateQuote({ ...first, width: 0, height: 0 }, "Reference: QR-2026-04")?.box).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("which pages a model is shown as an image", () => {
|
||||
it("is a scan, a page the reader was unsure of, or a page with next to no text", () => {
|
||||
expect(isWeakPage(first)).toBe(false)
|
||||
expect(isWeakPage(second)).toBe(true) // source: ocr
|
||||
expect(isWeakPage({ ...first, lines: first.lines.map((line) => ({ ...line, confidence: 0.5 })) })).toBe(true)
|
||||
expect(isWeakPage({ ...first, text: " Hi " })).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, it } from "vitest"
|
||||
import { buildWorkbook } from "../core/xlsx.js"
|
||||
import { unzip } from "./support/unzip.js"
|
||||
|
||||
describe("the workbook writer", () => {
|
||||
const book = buildWorkbook("Approved lines", [
|
||||
["Label", "Value", "Page"],
|
||||
["Total", "12,50", 2],
|
||||
])
|
||||
const files = unzip(book)
|
||||
|
||||
it("writes the parts a spreadsheet application needs to open the file", () => {
|
||||
expect([...files.keys()].sort()).toEqual([
|
||||
"[Content_Types].xml",
|
||||
"_rels/.rels",
|
||||
"xl/_rels/workbook.xml.rels",
|
||||
"xl/styles.xml",
|
||||
"xl/workbook.xml",
|
||||
"xl/worksheets/sheet1.xml",
|
||||
])
|
||||
expect(files.get("xl/workbook.xml")).toContain('<sheet name="Approved lines" sheetId="1" r:id="rId1"/>')
|
||||
})
|
||||
|
||||
it("keeps text as text and a number as a number, in the cell it was given", () => {
|
||||
const sheet = files.get("xl/worksheets/sheet1.xml") ?? ""
|
||||
expect(sheet).toContain('<c r="A1" t="inlineStr" s="1"><is><t xml:space="preserve">Label</t></is></c>')
|
||||
expect(sheet).toContain('<c r="B2" t="inlineStr"><is><t xml:space="preserve">12,50</t></is></c>')
|
||||
expect(sheet).toContain('<c r="C2"><v>2</v></c>')
|
||||
})
|
||||
|
||||
it("writes a formula-looking value as inert text, never as a formula", () => {
|
||||
const sheet =
|
||||
unzip(buildWorkbook("S", [['=HYPERLINK("http://x")', "+1+1", "@SUM(A1)", "-2+3"]])).get(
|
||||
"xl/worksheets/sheet1.xml",
|
||||
) ?? ""
|
||||
expect(sheet).not.toContain("<f>")
|
||||
expect(sheet.match(/t="inlineStr"/g)).toHaveLength(4)
|
||||
expect(sheet).toContain("=HYPERLINK("http://x")")
|
||||
})
|
||||
|
||||
it("escapes markup and drops characters XML cannot carry", () => {
|
||||
const sheet = unzip(buildWorkbook("S", [["a<b>&\u0000c\u0007d"]])).get("xl/worksheets/sheet1.xml") ?? ""
|
||||
expect(sheet).toContain("a<b>&cd")
|
||||
expect([...sheet].some((char) => char.charCodeAt(0) < 9)).toBe(false)
|
||||
})
|
||||
|
||||
it("refuses a sheet name Excel would reject", () => {
|
||||
for (const name of ["", "a/b", "x".repeat(32), "[a]", "a:b"]) {
|
||||
expect(() => buildWorkbook(name, [])).toThrow(/sheet name/)
|
||||
}
|
||||
})
|
||||
|
||||
it("is byte-for-byte reproducible, so an export of the same lines is the same file", () => {
|
||||
expect(Buffer.from(buildWorkbook("S", [["a"]])).equals(Buffer.from(buildWorkbook("S", [["a"]])))).toBe(true)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user