// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
// The documents extraction eval on the second extractor: the platform's Document Items run (ADR-069),
// behind the same `LineExtractor` port, scored on the same golden set, by the same scorer, against the
// same offline thresholds as the model extractor (documents.eval.test.ts).
//
// Offline, the fake platform (`@conusai-kit/testing`) plays the platform: each case is a job whose
// document is the case's OCR document, and its items run answers what the deterministic reader of
// test/support/fake-model.ts finds on the pages the platform reads (not drawings, not empty pages, not
// repeated lines), each item cited by its line and box. That measures this module's side of the seam:
// the request, the mapping of `conusai.documents.items/v1` to lines, the second verification of every
// quote against the page, the skipped pages, the scoring. The negative controls prove it can fail.
//
// Live there is no job to run a profile over (the golden set is text, not files), so the live items
// eval is the customer repository's own, against its private set.

import { platformItemsExtractor, sdkDocumentOcr } from "@app/modules/documents"
import { createDocumentReader } from "@conusai-kit/platform"
import { type FakeItemsResponder, startFakePlatform } from "@conusai-kit/testing/fake-platform"
import { afterAll, beforeAll, describe, expect, it } from "vitest"
import { readDocumentSchema } from "../../modules/documents/core/read-document.js"
import { keyValueLines } from "../../test/support/fake-model.js"
import {
  aggregate,
  type CaseResult,
  failuresOf,
  type GoldenCase,
  loadCases,
  loadThresholds,
  noCache,
  readDocumentOf,
  report,
  scoreCase,
} from "./harness.js"

const live = process.env.EVAL_LIVE === "1"
const cases = loadCases()
const thresholds = loadThresholds()

/** What the platform reads off a stored document: every field of a page it does not leave out, cited by its line. */
const deterministicItems: FakeItemsResponder = ({ document, sha256 }) => {
  const read = readDocumentSchema.parse(document)
  const items: Array<Record<string, unknown>> = []
  for (const page of read.pages) {
    if (page.class === "drawing" || page.class === "empty") continue
    const lines = page.lines.filter((line) => !line.repeated)
    for (const [n, field] of keyValueLines(lines.map((line) => line.text).join("\n")).entries()) {
      const line = lines.find((candidate) => candidate.text.trim() === field.quote)
      // The service numbers a line 1-based in reading order, the n of `p{page}-l{n}`
      // (crates/conusai-ocr/src/items/lines.rs). A field no line holds cites no line, and its id names none.
      const number = line ? line.order + 1 : undefined
      items.push({
        id: number === undefined ? `p${page.index + 1}-field${n + 1}` : `p${page.index + 1}-l${number}`,
        section: null,
        position: null,
        name: field.label,
        class: "other",
        unit: { text: "", code: null },
        // A key/value field's value is free text, not a quantity: the platform answers no decimal and says why.
        quantity: { value: null, text: field.value, column: "", refusal: "not_a_number" },
        spec: null,
        citations: [
          {
            kind: "pdf_page",
            file_sha256: sha256,
            page: page.index + 1,
            lines: number === undefined ? [] : [number],
            bbox: line ? [...line.bbox] : null,
            quote: field.quote,
          },
        ],
        checks: [{ check: "quote_verbatim", status: "passed" }],
        agreement: { status: "auto_accepted", reader: true, model: true },
        method: "reader_and_model",
        confidence: 0.97,
      })
    }
  }
  return items
}

describe.skipIf(live)("the platform items extractor, offline against the fake platform", () => {
  let platform: Awaited<ReturnType<typeof startFakePlatform>>
  const env = () => ({ CONUSAI_API_URL: `${platform.url}/api`, CONUSAI_API_TOKEN: "eval" })

  beforeAll(async () => {
    platform = await startFakePlatform()
  })
  afterAll(async () => {
    await platform.close()
  })

  /**
   * Each case as a finished job of the fake platform, then the extractor over that job. Every pass submits
   * new jobs: a job and its items run are idempotent, so a pass over the same jobs would read the runs the
   * last pass finished, whatever the platform answers now.
   */
  let pass = 0
  const evaluate = async (responder: FakeItemsResponder) => {
    pass += 1
    platform.ocr.setItems(responder)
    const reader = createDocumentReader(env())
    const extractor = platformItemsExtractor({ items: reader.items, intervalMs: 1 })
    const ocr = sdkDocumentOcr(reader, { intervalMs: 1 })
    const results: CaseResult[] = []
    for (const golden of cases) {
      platform.ocr.setDocument({ schema_version: "conusai.ocr.document/v1", ...readDocumentOf(golden) })
      const { jobId } = await ocr.submit(new TextEncoder().encode(golden.id), {
        filename: `${golden.id}.pdf`,
        idempotencyKey: `items.${pass}.${golden.id}`,
      })
      await ocr.wait(jobId)
      const extraction = await extractor.extract({
        document: readDocumentOf(golden),
        ocrJobId: jobId,
        pageImage: async () => undefined,
        cache: noCache,
      })
      results.push(scoreCase(golden, extraction))
    }
    return { results, metrics: aggregate(results) }
  }

  it("scores line recall, precision and quote verification above the same thresholds as the model", async () => {
    const { results, metrics } = await evaluate(deterministicItems)
    console.log(`platform items\n${report(results, metrics)}`)
    expect(failuresOf(metrics, thresholds.offline)).toEqual([])
    expect(results.flatMap((result) => result.skippedWrongly)).toEqual([])
    expect(metrics.expected).toBeGreaterThan(30)
    const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as { items: Array<{ profile: string }> }
    expect(state.items.length).toBeGreaterThanOrEqual(cases.length)
    expect(new Set(state.items.map((run) => run.profile))).toEqual(new Set(["invoice"]))
  })

  it("fails when the platform's quotes are not on the page, even when it calls them exact", async () => {
    const { metrics } = await evaluate(async (request) =>
      (await deterministicItems(request)).map((item) => {
        for (const citation of item.citations as Array<{ quote: string }>)
          citation.quote = citation.quote.replace(/\d/g, (digit) => String((Number(digit) + 1) % 10))
        return item
      }),
    )
    expect(metrics.verification).toBeLessThan(thresholds.offline.verification)
  })

  it("fails when the platform finds nothing, and when it invents items", async () => {
    expect((await evaluate(() => [])).metrics.recall).toBe(0)
    const { metrics } = await evaluate(async (request) => {
      const items = await deterministicItems(request)
      const [first] = items
      return first
        ? [
            ...items,
            {
              ...first,
              id: "invented",
              name: "Invented",
              quantity: { value: "1", text: "1", column: "", refusal: null },
            },
          ]
        : items
    })
    expect(metrics.precision).toBeLessThan(thresholds.offline.precision)
  })
})

/** The golden set has a case for each extractor to fail on, so neither passes by finding nothing. */
describe("the golden set as the items extractor sees it", () => {
  it("has fields on pages the platform reads", () => {
    const readable = (golden: GoldenCase) =>
      readDocumentOf(golden).pages.filter((page) => page.class !== "drawing" && page.class !== "empty").length
    expect(cases.some((golden) => readable(golden) > 0)).toBe(true)
  })
})