template-app/test/documents-flow.test.ts

// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The document-review flow end to end on the fake platform (plan item AT2.3): an uploaded file is
// read by an OCR job, becomes proposals, a person approves, edits or rejects each line, and only
// approved lines are exported. Every step runs through the kernel's caller, the files module's
// processor point, the proposals module's writer and the outbox relay's behaviour, so what is proven
// is the wiring and not a unit.

import { readFileSync } from "node:fs"
import { type LineView, lines as linesTable, REVIEW_PROCEDURES, type ReadDocument } from "@app/modules/documents"
import { AppError } from "@conusai-kit/core"
import { orgMemberships } from "@conusai-kit/tenancy"
import { uploadViaProcedures } from "@conusai-kit/testing"
import { afterEach, describe, expect, it } from "vitest"
import { sheetRows } from "../modules/documents/__tests__/support/unzip.js"
import {
  bytesOf,
  closeApps,
  type DocumentsApp,
  makeDocumentsApp,
  type SeededProject,
  uploadAndProcess,
} from "./support/documents-app.js"

const document = JSON.parse(
  readFileSync(new URL("../modules/documents/__tests__/fixtures/ocr-document.json", import.meta.url), "utf8"),
) as ReadDocument & Record<string, unknown>

afterEach(closeApps)

interface Detail {
  document: {
    documentId: string | null
    status: string
    failureCode: string | null
    pending: number
    approved: number
    pageCount: number | null
    skippedPages: unknown[]
  }
  lines: LineView[]
}

async function fresh(): Promise<{ app: DocumentsApp; project: SeededProject }> {
  const app = await makeDocumentsApp({ document })
  return { app, project: await app.seedProject("acme", "owner@acme.test") }
}

const detail = async (project: SeededProject, fileId: string): Promise<Detail> =>
  (await project.call.callByName("documents.document.get", { ...project.slugs, fileId })) as Detail

const decide = (project: SeededProject, procedure: string, input: Record<string, unknown>) =>
  project.call.callByName(procedure, { ...project.slugs, ...input })

const exportRows = async (project: SeededProject, fileId: string) => {
  const made = (await decide(project, "documents.export.create", { fileId })) as {
    body: string
    lineCount: number
    filename: string
  }
  return { ...made, rows: sheetRows(Buffer.from(made.body, "base64")) }
}

const refusal = async (promise: Promise<unknown>): Promise<AppError> => {
  try {
    await promise
  } catch (error) {
    if (error instanceof AppError) return error
    throw error
  }
  throw new Error("expected a refusal")
}

describe("uploading a document", () => {
  it("answers before OCR runs: the request enqueues, and the worker reads", async () => {
    const { app, project } = await fresh()
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "report.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4 fixture"),
    })
    // The upload has returned and the reader has not been asked anything.
    expect(app.ocr.submitted).toHaveLength(0)
    expect(app.queue.of("documents.ocr")).toHaveLength(0)
    // The stored file's event is what enqueues the job, once.
    await app.deliver()
    expect(app.queue.of("documents.ocr")).toHaveLength(1)
    expect(app.ocr.submitted).toHaveLength(0)
    await app.runJobs()
    expect(app.ocr.submitted).toHaveLength(1)
    const inbox = (await project.call.callByName("notifications.inbox.list", { org: project.orgSlug })) as {
      items: Array<{ code: string; href: string | null }>
    }
    expect(inbox.items.filter((item) => item.code === "job_completed")).toHaveLength(1)
    expect(inbox.items.find((item) => item.code === "job_completed")?.href).toBe(
      `/${project.orgSlug}/${project.workspaceSlug}/${project.projectSlug}/documents/${file.fileId}`,
    )
  })

  it("is not read by this module when it is not a document type it reads", async () => {
    const { app, project } = await fresh()
    await uploadViaProcedures(project.call, project.slugs, {
      filename: "notes.csv",
      contentType: "text/csv",
      bytes: bytesOf("a,b"),
    })
    await app.deliver()
    expect(app.queue.of("documents.ocr")).toHaveLength(0)
  })

  it("lists an upload nobody has read yet as queued", async () => {
    const { app, project } = await fresh()
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "report.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4"),
    })
    const { documents } = (await decide(project, "documents.document.list", {})) as {
      documents: Array<{ fileId: string; status: string }>
    }
    expect(documents).toEqual([expect.objectContaining({ documentId: null, fileId: file.fileId, status: "queued" })])
    expect(app.ocr.submitted).toHaveLength(0)
  })
})

describe("the live collection's read-back", () => {
  it("reads a document again by the id of its row, with its counts, as the list shows it", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const { documents } = (await decide(project, "documents.document.list", {})) as {
      documents: Array<{ documentId: string | null; fileId: string }>
    }
    const listed = documents.find((entry) => entry.fileId === file.fileId)
    expect(listed?.documentId).toEqual(expect.any(String))
    const { items } = (await decide(project, "documents.document.getMany", { ids: [listed?.documentId] })) as {
      items: unknown[]
    }
    expect(items).toEqual([listed])
  })

  it("answers nothing for an id of another project or of nothing", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const documentId = (await detail(project, file.fileId)).document.documentId
    const other = await app.seedProject("bravo", "owner@bravo.test")
    const ask = (asker: SeededProject, ids: string[]) =>
      asker.call.callByName("documents.document.getMany", { ...asker.slugs, ids }) as Promise<{ items: unknown[] }>
    expect((await ask(other, [documentId as string])).items).toEqual([])
    expect((await ask(project, ["00000000-0000-0000-0000-000000000000"])).items).toEqual([])
  })
})

describe("the extraction", () => {
  it("turns the fixture document into one pending line per labelled line, each with its page and verbatim quote", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const { document: view, lines } = await detail(project, file.fileId)

    // the fixture's third page is a drawing: left out, and said so on the document
    expect(view).toMatchObject({
      status: "extracted",
      pending: 4,
      approved: 0,
      pageCount: 3,
      skippedPages: [{ page: 3, pageClass: "drawing" }],
    })
    expect(lines.map((line) => [line.page, line.label, line.value, line.status])).toEqual([
      [1, "Reference", "QR-2026-04", "pending"],
      [1, "Issued on", "2026-04-01", "pending"],
      [2, "Total amount", "1 250,00 EUR", "pending"],
      [2, "Signed by", "A. Example", "pending"],
    ])
    for (const line of lines) {
      expect(document.pages[line.page - 1]?.text).toContain(line.quote)
      expect(line.box).not.toBeNull()
      // The line says which engine and extractor produced it: the Art. 50 disclosure's subject.
      expect(line.model).toBe("fake-model")
      expect(line.source).toMatchObject({
        ocrEngine: { name: "fixture-engine", version: document.engine.version },
        model: "fake-model",
        fileSha256: expect.stringMatching(/^[0-9a-f]{64}$/),
      })
      expect(line.promptVersion).toBe("extract-lines.v2")
    }
  })

  it("submits the document once, with a key that names the file and its content", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    expect(app.ocr.submitted).toHaveLength(1)
    expect(app.ocr.submitted[0]?.idempotencyKey).toMatch(
      new RegExp(`^documents\\.ocr\\.${file.fileId}\\.[0-9a-f]{64}$`),
    )
  })

  it("writes nothing the reviewer has not approved", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    expect(
      await app.data
        .forScope({ level: "project", actor: project.owner, ...scopeIds(project) })
        .of(linesTable)
        .count(),
    ).toBe(0)
    // and so an export has nothing to say yet
    const error = await refusal(decide(project, "documents.export.create", { fileId: file.fileId }))
    expect(error.status).toBe(409)
  })
})

describe("retrying the job", () => {
  it("a job run twice adds no proposal and submits no second document", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const job = app.queue.jobs.find((entry) => entry.job === "documents.ocr")
    expect(job).toBeDefined()
    // The queue delivers the same job again (a lapsed claim): run its handler once more.
    const again = app.queue.jobs.length
    await app.queue.enqueue(job?.scope as never, "documents.ocr", job?.payload as never)
    expect(app.queue.jobs).toHaveLength(again + 1)
    await app.runJobs()
    expect(app.ocr.submitted).toHaveLength(1)
    expect((await detail(project, file.fileId)).lines).toHaveLength(4)
  })

  it("a transient failure while waiting resumes the stored OCR job instead of submitting another", async () => {
    const { app, project } = await fresh()
    app.ocr.failWaitOnce(new Error("the platform is unreachable"))
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "report.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4"),
    })
    await app.deliver()
    await expect(app.runJobs()).rejects.toThrow("unreachable")
    expect((await detail(project, file.fileId)).document.status).toBe("processing")

    // pg-boss retries a job that throws: the same payload, the same scope.
    const job = app.queue.jobs.find((entry) => entry.job === "documents.ocr")
    await app.queue.enqueue(job?.scope as never, "documents.ocr", job?.payload as never, { key: "retry" })
    await app.runJobs()
    expect(app.ocr.submitted).toHaveLength(1)
    expect(app.ocr.waited).toHaveLength(2)
    expect((await detail(project, file.fileId)).document).toMatchObject({ status: "extracted", pending: 4 })
  })

  it("marks the document failed, with a code and no proposals, when the platform says the job failed", async () => {
    const { app, project } = await fresh()
    app.platform.ocr.failJobs(true)
    const file = await uploadAndProcess(app, project)
    const { document: view, lines } = await detail(project, file.fileId)
    expect(view).toMatchObject({ status: "failed", failureCode: "ocr_failed" })
    expect(lines).toEqual([])
  })

  it("marks the document failed when the stored bytes are not the ones that were uploaded", async () => {
    const { app, project } = await fresh()
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "report.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4 original"),
    })
    for (const [key, object] of app.store.objects)
      app.store.objects.set(key, { ...object, bytes: new TextEncoder().encode("tampered") })
    await app.deliver()
    await app.runJobs()
    expect((await detail(project, file.fileId)).document).toMatchObject({
      status: "failed",
      failureCode: "file_changed",
    })
    const inbox = (await project.call.callByName("notifications.inbox.list", { org: project.orgSlug })) as {
      items: Array<{ code: string }>
    }
    expect(inbox.items.filter((item) => item.code === "job_failed")).toHaveLength(1)
    expect(app.ocr.submitted).toHaveLength(0)
  })
})

describe("the review", () => {
  it("approving a line writes the record and one audit event; the other lines stay pending", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    await decide(project, REVIEW_PROCEDURES.approve, {
      proposalId: first?.lineId,
      expectedVersion: first?.expectedVersion,
    })
    await app.deliver()

    const after = await detail(project, file.fileId)
    expect(after.document).toMatchObject({ pending: 3, approved: 1 })
    expect(after.lines.find((line) => line.lineId === first?.lineId)?.status).toBe("approved")
    const approvals = app.audit.events.filter((event) => event.action === "proposals.proposal_approved")
    expect(approvals).toHaveLength(1)
    // ids and hashes only: no label, value or quote in an audit event
    expect(JSON.stringify(approvals[0])).not.toContain("QR-2026-04")
  })

  it("rejecting a line changes no record, and the line stays rejected with its audit event", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: first?.lineId, reason: "not a field" })
    await app.deliver()

    const after = await detail(project, file.fileId)
    expect(after.lines.find((line) => line.lineId === first?.lineId)?.status).toBe("rejected")
    expect(after.document).toMatchObject({ pending: 3, approved: 0 })
    expect(app.audit.events.filter((event) => event.action === "proposals.proposal_rejected")).toHaveLength(1)
    // nothing was written to the approved table: the export has nothing to say
    expect((await refusal(decide(project, "documents.export.create", { fileId: file.fileId }))).status).toBe(409)
  })

  it("an edit replaces the line with the reviewer's words, which are then approved like any other", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [, second] = (await detail(project, file.fileId)).lines
    const edited = (await decide(project, "documents.line.edit", {
      lineId: second?.lineId,
      expectedVersion: second?.expectedVersion,
      label: "Issue date",
      value: "2026-04-02",
    })) as LineView
    // the same line, with the reviewer's words in it
    expect(edited).toMatchObject({
      lineId: second?.lineId,
      expectedVersion: (second?.expectedVersion ?? 0) + 1,
      label: "Issue date",
      value: "2026-04-02",
      edited: true,
      status: "pending",
    })
    // the quote still names the words the machine read, not the reviewer's
    expect(edited.quote).toBe(second?.quote)
    const stale = await refusal(
      decide(project, "documents.line.edit", {
        lineId: second?.lineId,
        expectedVersion: second?.expectedVersion,
        label: "Stale correction",
        value: "outdated",
      }),
    )
    expect(stale.status).toBe(409)

    const listed = (await detail(project, file.fileId)).lines
    expect(listed).toHaveLength(4)
    expect(listed.find((line) => line.lineId === second?.lineId)).toMatchObject({ label: "Issue date", edited: true })
    await app.deliver()
    // one audit event for the edit, with no word of it
    const audited = app.audit.events.filter((event) => event.action === "proposals.proposal_edited")
    expect(audited).toHaveLength(1)
    expect(JSON.stringify(audited[0])).not.toContain("Issue date")

    await decide(project, REVIEW_PROCEDURES.approve, {
      proposalId: edited.lineId,
      expectedVersion: edited.expectedVersion,
    })
    const { rows } = await exportRows(project, file.fileId)
    expect(rows).toEqual([
      ["Label", "Value", "Page", "Source quote", "Edited by a reviewer"],
      ["Issue date", "2026-04-02", 1, "Issued on: 2026-04-01", "yes"],
    ])
  })

  it("a line that has been decided cannot be edited", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: first?.lineId })
    const error = await refusal(
      decide(project, "documents.line.edit", {
        lineId: first?.lineId,
        expectedVersion: first?.expectedVersion,
        label: "x",
        value: "y",
      }),
    )
    expect(error.status).toBe(409)
  })

  it("approves every pending line in one decision", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const lines = (await detail(project, file.fileId)).lines
    const ids = lines.map((line) => line.lineId)
    const expectations = Object.fromEntries(
      ids.map((id, index) => [id, { expectedVersion: lines[index]?.expectedVersion }]),
    )
    await decide(project, REVIEW_PROCEDURES.approveMany, { proposalIds: ids, expectations })
    expect((await detail(project, file.fileId)).document).toMatchObject({ pending: 0, approved: 4 })
  })
})

describe("the export", () => {
  it("contains the approved lines and only those, as an Excel workbook", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project, "quarterly.report.pdf")
    const lines = (await detail(project, file.fileId)).lines
    await decide(project, REVIEW_PROCEDURES.approve, {
      proposalId: lines[2]?.lineId,
      expectedVersion: lines[2]?.expectedVersion,
    })
    await decide(project, REVIEW_PROCEDURES.approve, {
      proposalId: lines[0]?.lineId,
      expectedVersion: lines[0]?.expectedVersion,
    })
    await decide(project, REVIEW_PROCEDURES.reject, { proposalId: lines[1]?.lineId })

    const made = await exportRows(project, file.fileId)
    expect(made.filename).toBe("quarterly.report-approved.xlsx")
    expect(made.lineCount).toBe(2)
    expect(made.rows.slice(1)).toEqual([
      ["Reference", "QR-2026-04", 1, "Reference: QR-2026-04", "no"],
      ["Total amount", "1 250,00 EUR", 2, "Total amount: 1 250,00 EUR", "no"],
    ])
  })

  it("writes one audit event naming no content", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    await decide(project, REVIEW_PROCEDURES.approve, {
      proposalId: first?.lineId,
      expectedVersion: first?.expectedVersion,
    })
    await decide(project, "documents.export.create", { fileId: file.fileId })
    // The transport records a procedure's declared audit event (apps/web router); here the
    // contract's own declaration is the proof, since no transport runs.
    expect(app.registry.procedures.get("documents.export.create")?.def.audit).toBe("documents.export_created")
    expect(app.registry.procedures.get("documents.line.edit")?.def.audit).toBe("documents.line_edited")
  })
})

describe("who may do what", () => {
  it("lets a viewer read a document and refuses them (404, as a stranger) an edit, an approval and an export", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    const viewer = await app.addMember(project, "viewer@acme.test", "viewer")

    const read = (await viewer.call.callByName("documents.document.get", {
      ...project.slugs,
      fileId: file.fileId,
    })) as Detail
    expect(read.lines).toHaveLength(4)
    for (const [name, input] of [
      [
        "documents.line.edit",
        { lineId: first?.lineId, expectedVersion: first?.expectedVersion, label: "x", value: "y" },
      ],
      [REVIEW_PROCEDURES.approve, { proposalId: first?.lineId, expectedVersion: first?.expectedVersion }],
      ["documents.export.create", { fileId: file.fileId }],
    ] as const) {
      const error = await refusal(viewer.call.callByName(name, { ...project.slugs, ...input }))
      // The kernel answers a member who lacks the permission exactly as it answers a stranger
      // (spec §3.3): 404, so a refusal does not say which procedures exist for whom.
      expect(error.status, name).toBe(404)
    }
    expect((await detail(project, file.fileId)).document.approved).toBe(0)
  })

  it("lets a member review: the permission of the line's writer is the member role's", async () => {
    const { app, project } = await fresh()
    const file = await uploadAndProcess(app, project)
    const [first] = (await detail(project, file.fileId)).lines
    const member = await app.addMember(project, "member@acme.test", "member")
    await member.call.callByName(REVIEW_PROCEDURES.approve, {
      ...project.slugs,
      proposalId: first?.lineId,
      expectedVersion: first?.expectedVersion,
    })
    expect((await detail(project, file.fileId)).document.approved).toBe(1)
  })
})

const scopeIds = (project: SeededProject) => ({
  orgId: project.orgId,
  workspaceId: project.workspaceId,
  projectId: project.projectId,
})

describe("current worker authority before provider costs", () => {
  it("keeps default missing native evidence off and persists an honest final outcome", async () => {
    const app = await makeDocumentsApp({ document, nativeAdmission: false })
    const project = await app.seedProject("off", "off@example.test")
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "off.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4 synthetic"),
    })
    await app.deliver()
    await app.runJobs({ finalAttempt: true })
    expect(app.ocr.submitted).toHaveLength(0)
    expect(app.model.requests).toHaveLength(0)
    expect((await detail(project, file.fileId)).document).toMatchObject({
      status: "failed",
      failureCode: "processing_exhausted",
    })
  })
  it("rechecks the original uploader's current role instead of forging a human actor", async () => {
    const { app, project } = await fresh()
    await uploadViaProcedures(project.call, project.slugs, {
      filename: "revoked.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4 synthetic"),
    })
    await app.deliver()
    const repo = app.data.forScope({ level: "org", orgId: project.orgId, actor: project.owner }).of(orgMemberships)
    const member = (await repo.list({ limit: 1 })).items[0]
    if (!member) throw new Error("the uploader fixture must have an organization membership")
    await repo.update(member.id, { role: "viewer" }, member.version)
    await expect(app.runJobs()).rejects.toMatchObject({ status: 403 })
    expect(app.ocr.submitted).toHaveLength(0)
    expect(app.model.requests).toHaveLength(0)
  })
  it("a lost claim never makes a provider call or marks a permanent failure", async () => {
    const { app, project } = await fresh()
    const file = await uploadViaProcedures(project.call, project.slugs, {
      filename: "lease.pdf",
      contentType: "application/pdf",
      bytes: bytesOf("%PDF-1.4 synthetic"),
    })
    await app.deliver()
    const abort = new AbortController()
    abort.abort()
    await expect(app.runJobs({ finalAttempt: true, signal: abort.signal })).rejects.toBeDefined()
    expect(app.ocr.submitted).toHaveLength(0)
    expect(app.model.requests).toHaveLength(0)
    expect((await detail(project, file.fileId)).document.status).not.toBe("failed")
  })
})