ConusAI vendor v0.1.0-beta.114

2026-11-14 09:56:45 +00:00
commit 1ce8d23c35
867 changed files with 122104 additions and 0 deletions
@@ -0,0 +1,309 @@
+// The automatic PDF intake (template lesson core-11): an uploaded PDF of an opted-in project is read by
+// the platform's document reader and indexed passage by passage into the opted-in collection, for the
+// administrator who turned it on, whose permission is checked again before each step that leaves the
+// process. A project that is not opted in, or has knowledge off, reads nothing; removing the document
+// tombstones every passage it indexed, even once the file itself is gone.
+import type { TenantScope } from "@conusai-kit/core"
+import { createMemoryDataSource } from "@conusai-kit/data"
+import { DEFAULT_FILES_POLICY, files, filesProcessors, sha256Hex } from "@conusai-kit/files"
+import { fakeObjectStore } from "@conusai-kit/testing"
+import { describe, expect, it } from "vitest"
+import {
+  AUTO_INDEX_JOB,
+  AUTO_INDEX_PROCESSOR_KEY,
+  autoIndex,
+  collections,
+  documents,
+  KNOWLEDGE_ENTITIES,
+  passageSets,
+} from "../contract/index.js"
+import { autoIndexUpload } from "../core/auto-index-job.js"
+import { removeDocument } from "../core/index-job.js"
+import type {
+  DocumentAiPlatform,
+  KnowledgeDeps,
+  KnowledgePassage,
+  KnowledgePassageIndex,
+  KnowledgeReader,
+} from "../core/ports.js"
+import { knowledgeModule } from "../module.js"
+
+const ADMIN = "01900000-0000-7000-0000-000000000009"
+const UPLOADER = "01900000-0000-7000-0000-000000000010"
+const scope: TenantScope = {
+  level: "project",
+  actor: { userId: "system:knowledge.autoindex", system: true },
+  orgId: "01900000-0000-7000-0000-000000000001",
+  workspaceId: "01900000-0000-7000-0000-000000000002",
+  projectId: "01900000-0000-7000-0000-000000000003",
+}
+const pdf = new TextEncoder().encode("%PDF-1.7 synthetic")
+const read: readonly KnowledgePassage[] = [
+  { page: 1, index: 0, blockIds: ["b1", "b2"], text: "First page, first passage" },
+  { page: 1, index: 1, blockIds: ["b3"], text: "First page, second passage" },
+  { page: 3, index: 0, blockIds: [], text: "Third page" },
+]
+
+function makeDeps(
+  options: { enabled?: boolean; reader?: KnowledgeReader; indexError?: Error; enablement?: DocumentAiPlatform } = {},
+) {
+  const data = createMemoryDataSource({ entities: [...KNOWLEDGE_ENTITIES, files] })
+  const store = fakeObjectStore()
+  const calls = {
+    reads: [] as Array<{ filename: string; idempotencyKey: string }>,
+    indexed: [] as Array<{ collectionName: string; fileId: string; passages: readonly KnowledgePassage[] }>,
+    removed: [] as Array<{ documentRef: string; passages: ReadonlyArray<{ page: number; index: number }> }>,
+    admitted: [] as string[],
+  }
+  const reader: KnowledgeReader = options.reader ?? {
+    async read(bytes, input) {
+      expect(bytes).toEqual(pdf)
+      calls.reads.push({ filename: input.filename, idempotencyKey: input.idempotencyKey })
+      return read
+    },
+  }
+  const passages: KnowledgePassageIndex = {
+    async reference(_scope, fileId) {
+      return `org/${scope.orgId}/${fileRow.storageKey}?${fileId}`
+    },
+    async index(_scope, input) {
+      expect(input.documentRef).toBe(`org/${scope.orgId}/${fileRow.storageKey}?${input.fileId}`)
+      calls.indexed.push({ collectionName: input.collectionName, fileId: input.fileId, passages: input.passages })
+      if (options.indexError) throw options.indexError
+    },
+    async remove(_scope, input) {
+      calls.removed.push({ documentRef: input.documentRef, passages: input.passages })
+    },
+  }
+  const deps: KnowledgeDeps = {
+    data,
+    files: { data, store, policy: DEFAULT_FILES_POLICY },
+    flags: { enabled: async () => options.enabled ?? true },
+    reader,
+    passages,
+    enablement: options.enablement ?? {
+      checkEligibility: async () => ({ ok: true }),
+      indexingQuota: async () => 1000,
+    },
+    beforeIndex: async (subject) => {
+      calls.admitted.push(subject)
+    },
+  }
+  return { data, store, deps, calls }
+}
+
+const fileRow = {
+  status: "stored" as const,
+  storageKey: `files/${scope.workspaceId}/${scope.projectId}/01900000-0000-7000-0000-000000000100`,
+  filename: "contract.pdf",
+  contentType: "application/pdf",
+  size: pdf.length,
+  sha256: "",
+  uploadedBy: UPLOADER,
+  storedAt: "2026-11-09T19:00:00.000Z",
+  uploadId: null,
+  declaredSize: null,
+  partSize: null,
+  deletingAt: null,
+}
+
+async function seed(fixture: ReturnType<typeof makeDeps>, options: { optIn?: boolean } = {}) {
+  const sha256 = await sha256Hex(pdf)
+  await fixture.data.forScope(scope).of(collections).insert({ name: "contracts", createdBy: ADMIN })
+  if (options.optIn ?? true)
+    await fixture.data
+      .forScope(scope)
+      .of(autoIndex)
+      .insert({ collectionName: "contracts", enabledBy: ADMIN, enabledAt: "2026-11-09T18:00:00.000Z" })
+  const file = await fixture.data
+    .forScope(scope)
+    .of(files)
+    .insert({ ...fileRow, sha256 })
+  await fixture.store.put(scope, { key: fileRow.storageKey, body: pdf, contentType: "application/pdf" })
+  return { fileId: file.id, storageKey: fileRow.storageKey, contentType: "application/pdf", sha256 }
+}
+
+/** The run's document, which these cases expect there to be. */
+async function indexedView<T>(view: T | null): Promise<T> {
+  if (view === null) throw new Error("the upload made no document")
+  return view
+}
+
+describe("the automatic PDF intake", () => {
+  it("reads an opted-in project's PDF and indexes its passages into the collection, for the administrator who opted in", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+
+    const done = await autoIndexUpload(fixture.deps, scope, upload)
+
+    expect(done).toMatchObject({ collectionName: "contracts", fileId: upload.fileId, status: "indexed" })
+    expect(fixture.calls.reads).toEqual([
+      { filename: "contract.pdf", idempotencyKey: `knowledge.autoindex.${upload.fileId}.${upload.sha256}` },
+    ])
+    expect(fixture.calls.indexed).toEqual([{ collectionName: "contracts", fileId: upload.fileId, passages: read }])
+    // The job runs for nobody else: the administrator's permission is what each step leaving the process checks.
+    expect(fixture.calls.admitted.length).toBeGreaterThan(0)
+    expect(new Set(fixture.calls.admitted)).toEqual(new Set([ADMIN]))
+    const [row] = (await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items
+    expect(row).toMatchObject({ filename: "contract.pdf", requestedBy: ADMIN, status: "indexed", error: null })
+    // What removal needs, kept beside the document: the file's reference and the passages per page.
+    const [set] = (await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items
+    expect(set).toMatchObject({
+      collectionName: "contracts",
+      fileId: upload.fileId,
+      documentRef: `org/${scope.orgId}/${fileRow.storageKey}?${upload.fileId}`,
+      pages: "2,0,1",
+    })
+  })
+
+  it("is the same work once: a redelivered job reads and indexes nothing again", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+    await autoIndexUpload(fixture.deps, scope, upload)
+    expect(await autoIndexUpload(fixture.deps, scope, upload)).toMatchObject({ status: "indexed" })
+    expect(fixture.calls.reads).toHaveLength(1)
+    expect(fixture.calls.indexed).toHaveLength(1)
+  })
+
+  it("reads nothing for a project that is not opted in, has knowledge off, or a file that is not a PDF", async () => {
+    const notOptedIn = makeDeps()
+    expect(await autoIndexUpload(notOptedIn.deps, scope, await seed(notOptedIn, { optIn: false }))).toBeNull()
+
+    const off = makeDeps({ enabled: false })
+    expect(await autoIndexUpload(off.deps, scope, await seed(off))).toBeNull()
+
+    const text = makeDeps()
+    const upload = await seed(text)
+    expect(await autoIndexUpload(text.deps, scope, { ...upload, contentType: "text/plain" })).toBeNull()
+
+    for (const fixture of [notOptedIn, off, text]) {
+      expect(fixture.calls.reads).toEqual([])
+      expect((await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items).toEqual([])
+    }
+  })
+
+  it("reads nothing once the administrator's permission is gone: the refusal is the job's", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+    const refused = Object.assign(new Error("permission_missing"), { status: 403 })
+    const deps: KnowledgeDeps = {
+      ...fixture.deps,
+      beforeIndex: async () => {
+        throw refused
+      },
+    }
+    await expect(autoIndexUpload(deps, scope, upload)).rejects.toBe(refused)
+    expect(fixture.calls.reads).toEqual([])
+    expect(fixture.calls.indexed).toEqual([])
+  })
+
+  it("records a PDF the reader cannot make passages of as failed, with a stable code", async () => {
+    const fixture = makeDeps({
+      reader: {
+        read: async () => {
+          throw Object.assign(new Error("block id over 256 characters"), { permanent: "not_indexable" })
+        },
+      },
+    })
+    const upload = await seed(fixture)
+    expect(await autoIndexUpload(fixture.deps, scope, upload)).toMatchObject({
+      status: "failed",
+      error: "not_indexable",
+    })
+    expect(fixture.calls.indexed).toEqual([])
+  })
+
+  it("keeps the passage layout of an index the service refused partway, so removal still tombstones it", async () => {
+    const fixture = makeDeps({ indexError: Object.assign(new Error("quota"), { permanent: "quota_exhausted" }) })
+    const upload = await seed(fixture)
+    const failed = await autoIndexUpload(fixture.deps, scope, upload)
+    expect(failed).toMatchObject({ status: "failed", error: "quota_exhausted" })
+    await removeDocument(fixture.deps, scope, (await indexedView(failed)).documentId)
+    expect(fixture.calls.removed.map((call) => call.passages.length)).toEqual([3])
+    expect((await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items).toEqual([])
+  })
+
+  it("reads nothing once the organization is no longer eligible or its indexing quota is spent", async () => {
+    const ineligible = makeDeps({
+      enablement: {
+        checkEligibility: async () => ({
+          ok: false,
+          reason: "feature_disabled",
+          detail: "the agent_document_tools flag is off",
+        }),
+        indexingQuota: async () => 1000,
+      },
+    })
+    expect(await autoIndexUpload(ineligible.deps, scope, await seed(ineligible))).toMatchObject({
+      status: "failed",
+      error: "feature_disabled",
+    })
+
+    const spent = makeDeps({
+      enablement: { checkEligibility: async () => ({ ok: true }), indexingQuota: async () => 0 },
+    })
+    expect(await autoIndexUpload(spent.deps, scope, await seed(spent))).toMatchObject({
+      status: "failed",
+      error: "quota_exhausted",
+    })
+
+    for (const fixture of [ineligible, spent]) {
+      expect(fixture.calls.reads).toEqual([])
+      expect(fixture.calls.indexed).toEqual([])
+    }
+  })
+
+  it("refuses to run where the enablement platform is not bound, rather than skip its checks", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+    const { enablement: _unbound, ...deps } = fixture.deps
+    await expect(autoIndexUpload(deps, scope, upload)).rejects.toThrow("enablement")
+    expect(fixture.calls.reads).toEqual([])
+  })
+
+  it("does not read a file whose bytes changed since the upload", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+    expect(await autoIndexUpload(fixture.deps, scope, { ...upload, sha256: "0".repeat(64) })).toBeNull()
+    expect(fixture.calls.reads).toEqual([])
+  })
+
+  it("tombstones every passage it indexed when the document is removed, even after the file is gone", async () => {
+    const fixture = makeDeps()
+    const upload = await seed(fixture)
+    const done = await autoIndexUpload(fixture.deps, scope, upload)
+    await fixture.data.forScope(scope).of(files).delete(upload.fileId)
+
+    await removeDocument(fixture.deps, scope, (await indexedView(done)).documentId)
+
+    expect(fixture.calls.removed).toEqual([
+      {
+        documentRef: `org/${scope.orgId}/${fileRow.storageKey}?${upload.fileId}`,
+        passages: [
+          { page: 1, index: 0 },
+          { page: 1, index: 1 },
+          { page: 3, index: 0 },
+        ],
+      },
+    ])
+    expect((await fixture.data.forScope(scope).of(documents).list({ limit: 10 })).items).toEqual([])
+    expect((await fixture.data.forScope(scope).of(passageSets).list({ limit: 10 })).items).toEqual([])
+  })
+})
+
+describe("the intake's place in the manifest", () => {
+  it("contributes a files processor for PDFs whose job the module runs", () => {
+    const contributions = (knowledgeModule.extends ?? []).filter((entry) => entry.point.id === filesProcessors.id)
+    expect(contributions.flatMap((entry) => entry.values)).toEqual([
+      { key: AUTO_INDEX_PROCESSOR_KEY, contentTypes: ["application/pdf"], job: AUTO_INDEX_JOB },
+    ])
+    expect(Object.keys(knowledgeModule.jobs ?? {})).toContain(AUTO_INDEX_JOB)
+  })
+
+  it("declares the platform operations the job uses: the knowledge service's and the document reader's", () => {
+    const operations = (service: string) =>
+      knowledgeModule.serviceRequirements?.find((requirement) => requirement.service === service)?.operations
+    expect(operations("knowledge")?.[AUTO_INDEX_JOB]).toEqual(["knowledge.collections:use"])
+    expect(operations("ocr")?.[AUTO_INDEX_JOB]).toEqual(["ocr:use"])
+  })
+})