ConusAI vendor v0.1.0-beta.114

modules/knowledge/core/auto-index-job.ts

@@ -0,0 +1,72 @@
+// The automatic PDF intake (template lesson core-11): what the worker runs for one uploaded file the
+// files module handed to this module's processor. A project that opted in (`knowledge_auto_index`)
+// gets the PDF indexed into its collection as a document like any other, requested by the
+// administrator who opted in: the job runs for nobody else, so that administrator's admission is what
+// each step leaving the process checks again (`beforeIndex`). Anything else reads nothing.
+
+import { AppError } from "@conusai-kit/core"
+import { getFile } from "@conusai-kit/files"
+import { AUTO_INDEX_CONTENT_TYPES, autoIndex, type DocumentView, documents } from "../contract/index.js"
+import { indexPassages } from "./index-job.js"
+import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+/** What the files module's processor job carries about the upload. */
+export interface AutoIndexUpload {
+  fileId: string
+  contentType: string
+  sha256: string
+}
+
+/**
+ * Indexes one upload of an opted-in project, or answers null and reads nothing: a type it does not read,
+ * a project that is not opted in or has knowledge off, a file that is gone, or one whose digest is not
+ * the upload's. A redelivered job finds the document row and resumes it, so the work is done once.
+ */
+export async function autoIndexUpload(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  upload: AutoIndexUpload,
+  options: { signal?: AbortSignal; finalAttempt?: boolean } = {},
+): Promise<DocumentView | null> {
+  if (!(AUTO_INDEX_CONTENT_TYPES as readonly string[]).includes(upload.contentType)) return null
+  const optIn = (await deps.data.forScope(scope).of(autoIndex).list({ limit: 1 })).items[0]
+  if (!optIn) return null
+  if (!(await knowledgeEnabled(deps, scope))) return null
+  const file = await storedFile(deps, scope, upload.fileId)
+  if (!file || file.sha256 !== upload.sha256) return null
+
+  const row = await deps.data.unitOfWork.run(scope, async (tx) => {
+    const existing = (
+      await tx.of(documents).list({
+        where: [
+          { field: "collectionName", op: "eq", value: optIn.collectionName },
+          { field: "fileId", op: "eq", value: upload.fileId },
+        ],
+        limit: 1,
+      })
+    ).items[0]
+    return (
+      existing ??
+      (await tx.of(documents).insert({
+        collectionName: optIn.collectionName,
+        fileId: upload.fileId,
+        filename: file.filename,
+        status: "queued",
+        error: null,
+        indexedAt: null,
+        requestedBy: optIn.enabledBy,
+      }))
+    )
+  })
+  return indexPassages(deps, scope, row.id, options)
+}
+
+/** The upload's file, or undefined once it is no longer stored (deleted, or on its way out). */
+async function storedFile(deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string) {
+  try {
+    return await getFile(deps.files, scope, fileId)
+  } catch (error) {
+    if (error instanceof AppError && error.status === 404) return undefined
+    throw error
+  }
+}

modules/knowledge/core/auto-index.ts

@@ -0,0 +1,46 @@
+// A project's automatic PDF indexing (template lesson core-11): an administrator opts the project in,
+// naming the collection its uploaded PDFs go into; no row is off. The administrator is recorded,
+// because the indexing job runs for nobody and re-checks that person's permission before each file.
+
+import { forbidden } from "@conusai-kit/core"
+import { autoIndex } from "../contract/index.js"
+import { requireCollection } from "./collections.js"
+import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+export interface AutoIndexView {
+  /** The collection uploaded PDFs are indexed into, or null when automatic indexing is off. */
+  collectionName: string | null
+}
+
+export async function getAutoIndex(deps: Pick<KnowledgeDeps, "data">, scope: KnowledgeScope): Promise<AutoIndexView> {
+  const row = (await deps.data.forScope(scope).of(autoIndex).list({ limit: 1 })).items[0]
+  return { collectionName: row?.collectionName ?? null }
+}
+
+/** Turns automatic indexing on into `collectionName` (which must exist), or off with `null`. */
+export async function setAutoIndex(
+  deps: KnowledgeDeps & { now?: () => number },
+  scope: KnowledgeScope,
+  collectionName: string | null,
+): Promise<AutoIndexView> {
+  if (collectionName !== null) {
+    // The tenant-wall read first: another project's collection answers NotFound, never a policy refusal.
+    await requireCollection(deps, scope, collectionName)
+    if (!(await knowledgeEnabled(deps, scope))) throw forbidden("knowledge is switched off for this project")
+  }
+  await deps.data.unitOfWork.run(scope, async (tx) => {
+    const row = (await tx.of(autoIndex).list({ limit: 1 })).items[0]
+    if (collectionName === null) {
+      if (row) await tx.of(autoIndex).delete(row.id)
+      return
+    }
+    const patch = {
+      collectionName,
+      enabledBy: scope.actor.userId,
+      enabledAt: new Date((deps.now ?? Date.now)()).toISOString(),
+    }
+    if (row) await tx.of(autoIndex).update(row.id, patch, row.version)
+    else await tx.of(autoIndex).insert(patch)
+  })
+  return { collectionName }
+}

modules/knowledge/core/collections.ts

@@ -0,0 +1,132 @@
+// The collections: list them with their document counts, create one. A collection is a named
+// bucket; its name is the slug every procedure takes, so a page never handles an id.
+
+import { conflict, notFound, validationFailed } from "@conusai-kit/core"
+import { type CollectionView, collections, documents } from "../contract/index.js"
+import type { KnowledgeDeps, KnowledgeScope } from "./ports.js"
+
+const PAGE = 200
+
+export async function listCollections(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+): Promise<{ collections: CollectionView[] }> {
+  const found = []
+  let cursor: string | undefined
+  for (;;) {
+    const page = await deps.data
+      .forScope(scope)
+      .of(collections)
+      .list({
+        order: [{ field: "createdAt", direction: "asc" }],
+        limit: PAGE,
+        ...(cursor === undefined ? {} : { cursor }),
+      })
+    found.push(...page.items)
+    if (!page.hasMore) break
+    cursor = page.cursor
+  }
+  const views: CollectionView[] = []
+  for (const row of found) {
+    const counts = await documentCounts(deps, scope, row.name)
+    views.push({
+      name: row.name,
+      documentCount: counts.total,
+      indexedCount: counts.indexed,
+      createdBy: row.createdBy,
+    })
+  }
+  return { collections: views }
+}
+
+async function documentCounts(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  collectionName: string,
+): Promise<{ total: number; indexed: number }> {
+  let total = 0
+  let indexed = 0
+  let cursor: string | undefined
+  for (;;) {
+    const page = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .list({
+        where: [{ field: "collectionName", op: "eq", value: collectionName }],
+        limit: PAGE,
+        ...(cursor === undefined ? {} : { cursor }),
+      })
+    for (const row of page.items) {
+      total += 1
+      if (row.status === "indexed") indexed += 1
+    }
+    if (!page.hasMore) break
+    cursor = page.cursor
+  }
+  return { total, indexed }
+}
+
+/**
+ * Creates a collection, idempotently: a name that already exists answers with the existing
+ * collection and writes nothing, so a retried request never audits twice.
+ */
+export async function createCollection(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  name: string,
+): Promise<CollectionView> {
+  const repos = deps.data.forScope(scope)
+  const existing = (await repos.of(collections).list({ where: [{ field: "name", op: "eq", value: name }], limit: 1 }))
+    .items[0]
+  if (existing) {
+    const counts = await documentCounts(deps, scope, name)
+    return {
+      name: existing.name,
+      documentCount: counts.total,
+      indexedCount: counts.indexed,
+      createdBy: existing.createdBy,
+    }
+  }
+  try {
+    const row = await deps.data.unitOfWork.run(scope, async (tx) => {
+      const inserted = await tx.of(collections).insert({ name, createdBy: scope.actor.userId })
+      return inserted
+    })
+    return { name: row.name, documentCount: 0, indexedCount: 0, createdBy: row.createdBy }
+  } catch (error) {
+    // A concurrent create with the same name lost the race: answer the winner's row.
+    if ((error as { status?: number }).status === 409) {
+      const winner = (await repos.of(collections).list({ where: [{ field: "name", op: "eq", value: name }], limit: 1 }))
+        .items[0]
+      if (winner) {
+        const counts = await documentCounts(deps, scope, name)
+        return {
+          name: winner.name,
+          documentCount: counts.total,
+          indexedCount: counts.indexed,
+          createdBy: winner.createdBy,
+        }
+      }
+    }
+    throw error
+  }
+}
+
+/** One collection by name, or NotFound: the resolver's own read is the membership check. */
+export async function requireCollection(deps: KnowledgeDeps, scope: KnowledgeScope, name: string) {
+  const row = (
+    await deps.data
+      .forScope(scope)
+      .of(collections)
+      .list({ where: [{ field: "name", op: "eq", value: name }], limit: 1 })
+  ).items[0]
+  if (!row) throw notFound("no such collection")
+  return row
+}
+
+export const unknownCollection = (name: string): never => {
+  throw validationFailed([{ path: "collectionName", message: "no such collection" }], `no collection ${name}`)
+}
+export const takenCollection = (): never => {
+  throw conflict("this collection name is taken")
+}

modules/knowledge/core/documents.ts

@@ -0,0 +1,255 @@
+// The documents: request an index (a web write that records the row and publishes the outbox
+// event the worker's subscriber turns into the index job), request a removal the same way, and
+// list a collection's documents with their progress states.
+
+import { AppError, conflict, forbidden, notFound, validationFailed } from "@conusai-kit/core"
+import { getFile, getFileStorage } from "@conusai-kit/files"
+import {
+  type DocumentView,
+  documentIndexRequested,
+  documentRemoveRequested,
+  documents,
+  INDEXABLE_CONTENT_TYPES,
+  MAX_INDEX_BYTES,
+} from "../contract/index.js"
+import { requireCollection } from "./collections.js"
+import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+const PAGE = 200
+
+export const documentView = (row: {
+  id: string
+  collectionName: string
+  fileId: string
+  filename: string
+  status: DocumentView["status"]
+  error: string | null
+  indexedAt: string | null
+  requestedBy: string
+}): DocumentView => ({
+  documentId: row.id,
+  collectionName: row.collectionName,
+  fileId: row.fileId,
+  filename: row.filename,
+  status: row.status,
+  error: row.error,
+  indexedAt: row.indexedAt,
+  requestedBy: row.requestedBy,
+})
+
+export async function listDocuments(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  collectionName: string,
+): Promise<{ documents: DocumentView[] }> {
+  const found = []
+  let cursor: string | undefined
+  for (;;) {
+    const page = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .list({
+        where: [{ field: "collectionName", op: "eq", value: collectionName }],
+        order: [{ field: "createdAt", direction: "desc" }],
+        limit: PAGE,
+        ...(cursor === undefined ? {} : { cursor }),
+      })
+    found.push(...page.items)
+    if (!page.hasMore) break
+    cursor = page.cursor
+  }
+  return { documents: found.map((row) => documentView(row)) }
+}
+
+/** The most ids one batch-get reads (contract C-05). */
+export const MAX_BATCH = 100
+
+/**
+ * The live collection's read-back: the documents with these ids, in whichever collection of the project.
+ * A row this caller cannot see is not in the answer: the repository's tenant wall leaves it out.
+ */
+export async function getManyDocuments(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  documentIds: readonly string[],
+): Promise<DocumentView[]> {
+  const page = await deps.data
+    .forScope(scope)
+    .of(documents)
+    .list({ where: [{ field: "id", op: "in", values: [...new Set(documentIds)] }], limit: MAX_BATCH })
+  return page.items.map((row) => documentView(row))
+}
+
+/**
+ * Asks for a document's indexing. The write is three things in one transaction: the durable row
+ * (the UI's progress states and the job's idempotency key), the outbox event the worker's
+ * subscriber enqueues the index job from, and nothing else — the call to the platform's knowledge
+ * service happens in the worker, never in a request (AGENTS.md: long work is a job).
+ *
+ * Idempotent by (collection, file): asking again for a pair that is queued or indexing answers
+ * the same row and publishes nothing; asking for an indexed or failed pair re-queues the one row,
+ * so a retry can never duplicate a document.
+ */
+export async function requestIndex(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  input: { collectionName: string; fileId: string },
+): Promise<DocumentView> {
+  // The tenant-wall reads come before the flag check: another tenant's ids answer NotFound, never
+  // a policy refusal, so ids do not leak existence behind a disabled flag.
+  await requireCollection(deps, scope, input.collectionName)
+  const file = await getFile(deps.files, scope, input.fileId)
+  if (!(INDEXABLE_CONTENT_TYPES as readonly string[]).includes(file.contentType)) {
+    throw validationFailed(
+      [{ path: "fileId", message: `${file.contentType} cannot be indexed; index a text file` }],
+      "this file type is not indexable",
+    )
+  }
+  if (file.size > MAX_INDEX_BYTES) {
+    throw validationFailed(
+      [{ path: "fileId", message: `the file is larger than ${MAX_INDEX_BYTES} bytes` }],
+      "the file is too large to index",
+    )
+  }
+  if (!(await knowledgeEnabled(deps, scope))) throw forbidden("knowledge is switched off for this project")
+
+  return deps.data.unitOfWork.run(scope, async (tx) => {
+    const existing = (
+      await tx.of(documents).list({
+        where: [
+          { field: "collectionName", op: "eq", value: input.collectionName },
+          { field: "fileId", op: "eq", value: input.fileId },
+        ],
+        limit: 1,
+      })
+    ).items[0]
+    if (existing && (existing.status === "queued" || existing.status === "indexing")) {
+      return documentView(existing)
+    }
+    const row =
+      existing !== undefined
+        ? await tx
+            .of(documents)
+            .update(
+              existing.id,
+              { status: "queued", error: null, indexedAt: null, requestedBy: scope.actor.userId },
+              existing.version,
+            )
+        : await tx.of(documents).insert({
+            collectionName: input.collectionName,
+            fileId: input.fileId,
+            filename: file.filename,
+            status: "queued",
+            error: null,
+            indexedAt: null,
+            requestedBy: scope.actor.userId,
+          })
+    await tx.publish(documentIndexRequested, {
+      orgId: scope.orgId,
+      workspaceId: scope.workspaceId,
+      projectId: scope.projectId,
+      documentId: row.id,
+      collectionName: input.collectionName,
+      fileId: input.fileId,
+    })
+    return documentView(row)
+  })
+}
+
+/** Asks for a document's removal: the row and the event in one transaction, the job in the worker. */
+export async function requestRemove(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  input: { collectionName: string; fileId: string; storageKey?: string },
+): Promise<DocumentView> {
+  const current = (
+    await deps.data
+      .forScope(scope)
+      .of(documents)
+      .list({
+        where: [
+          { field: "collectionName", op: "eq", value: input.collectionName },
+          { field: "fileId", op: "eq", value: input.fileId },
+        ],
+        limit: 1,
+      })
+  ).items[0]
+  // The tenant-wall read comes first, as in requestIndex: another tenant's ids answer NotFound.
+  if (!current) throw notFound("no such document")
+  if (current.status === "cancelled") return documentView(current)
+  // Every removal names the object it removes, resolved when it is asked for, not when the job runs: by then the
+  // file may be gone (F407). A deletion brings the key it had; a person's removal reads it from the stored file.
+  const storageKey = input.storageKey ?? (await storageKeyOf(deps, scope, input.fileId))
+  return deps.data.unitOfWork.run(scope, async (tx) => {
+    const row = (
+      await tx.of(documents).list({
+        where: [
+          { field: "collectionName", op: "eq", value: input.collectionName },
+          { field: "fileId", op: "eq", value: input.fileId },
+        ],
+        limit: 1,
+      })
+    ).items[0]
+    if (!row) throw notFound("no such document")
+    if (row.status === "cancelled") return documentView(row)
+    const next = await tx.of(documents).update(row.id, { status: "cancelled", error: null }, row.version)
+    await tx.publish(documentRemoveRequested, {
+      orgId: scope.orgId,
+      workspaceId: scope.workspaceId,
+      projectId: scope.projectId,
+      documentId: row.id,
+      collectionName: input.collectionName,
+      fileId: input.fileId,
+      storageKey,
+    })
+    return documentView(next)
+  })
+}
+
+/** The document row a job resumes by; NotFound when another tenant's id was named. */
+export async function documentById(deps: KnowledgeDeps, scope: KnowledgeScope, documentId: string) {
+  const row = await deps.data.forScope(scope).of(documents).get(documentId)
+  if (!row) throw notFound("no such document")
+  return row
+}
+
+/**
+ * The stored file's object key. A file on its way out (being deleted, or gone) is refused rather than removed
+ * unkeyed: its deletion publishes `FileDeleted` with the key, and that removal finds the document still active.
+ */
+async function storageKeyOf(deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string): Promise<string> {
+  try {
+    return (await getFileStorage(deps.files, scope, fileId)).storageKey
+  } catch (error) {
+    if (error instanceof AppError && error.status === 404) {
+      throw conflict("the file is being deleted; its indexed text is removed with it")
+    }
+    throw error
+  }
+}
+
+/** A source deletion revokes every collection's citation and durably requests cleanup.
+ * Does not require processing enablement: deleting retained data is independent of inference.
+ * `storageKey` is the key the deleted file had (`FileDeleted`): its row is gone, so the removal cannot look it up. */
+export async function requestSourceRemoval(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  fileId: string,
+  storageKey: string,
+): Promise<void> {
+  let cursor: string | undefined
+  for (;;) {
+    const page = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .list({
+        where: [{ field: "fileId", op: "eq", value: fileId }],
+        limit: PAGE,
+        ...(cursor === undefined ? {} : { cursor }),
+      })
+    for (const row of page.items)
+      await requestRemove(deps, scope, { collectionName: row.collectionName, fileId, storageKey })
+    if (!page.hasMore) break
+    cursor = page.cursor
+  }
+}

modules/knowledge/core/index-job.ts

@@ -0,0 +1,280 @@
+// The index job's body: what the worker runs for one document. Read the file's bytes, hand them to
+// the search service, and record the outcome on the durable row.
+// Transient failures (the service hiccups) throw, and the queue retries the job; the row the
+// retry resumes from makes the retry safe. Permanent failures are recorded on the row with a
+// stable code, so a person sees why a document failed and a retried request does not pay twice.
+// The automatic PDF intake (core-11) runs the same lifecycle with another send: the PDF is read into
+// passages and each passage is indexed on its own (`passageSend`).
+
+import { internal } from "@conusai-kit/core"
+import { downloadFile, getFile, readAll, sha256Hex } from "@conusai-kit/files"
+import {
+  type DocumentView,
+  documents,
+  INDEX_FAILURES,
+  type IndexFailure,
+  MAX_AUTO_INDEX_BYTES,
+  MAX_INDEX_BYTES,
+  passageSets,
+} from "../contract/index.js"
+import { documentById, documentView } from "./documents.js"
+import { type KnowledgeDeps, type KnowledgePassage, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+type DocumentRow = Awaited<ReturnType<typeof documentById>>
+
+/** What one run sends between the row's "indexing" and its outcome; it throws to fail or to retry. */
+type Send = (row: DocumentRow, signal: AbortSignal | undefined) => Promise<void>
+
+export async function indexDocument(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  documentId: string,
+  options: { signal?: AbortSignal; finalAttempt?: boolean } = {},
+): Promise<DocumentView> {
+  return runIndex(deps, scope, documentId, () => textSend(deps, scope), options)
+}
+
+/** The automatic PDF intake's run of one document: the PDF read into passages, each indexed on its own. */
+export async function indexPassages(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  documentId: string,
+  options: { signal?: AbortSignal; finalAttempt?: boolean } = {},
+): Promise<DocumentView> {
+  return runIndex(deps, scope, documentId, () => passageSend(deps, scope), options)
+}
+
+/** The whole file as text, to the search service. */
+function textSend(deps: KnowledgeDeps, scope: KnowledgeScope): Send {
+  const service = deps.search
+  if (!service) throw internal("the knowledge search service is not bound in this process")
+  return async (row, signal) => {
+    const bytes = await readFileBytes(deps, scope, row.fileId, MAX_INDEX_BYTES)
+    const text = decodeText(bytes)
+    await deps.beforeIndex?.(row.requestedBy)
+    signal?.throwIfAborted()
+    await service.index(scope, {
+      collectionName: row.collectionName,
+      fileId: row.fileId,
+      filename: row.filename,
+      bytes: new TextEncoder().encode(text),
+      ...(signal ? { signal } : {}),
+    })
+  }
+}
+
+/**
+ * A PDF, read by the platform's document reader into passages, each indexed as one item. What is about to
+ * be indexed is recorded before any of it is sent, so a run the service refuses partway can still be
+ * removed passage by passage. The read is keyed by the file and its digest: a retry reads the same job.
+ */
+function passageSend(deps: KnowledgeDeps, scope: KnowledgeScope): Send {
+  const { reader, passages, enablement } = deps
+  if (!reader || !passages) throw internal("the document reader and the passage index are not bound in this process")
+  if (!enablement) throw internal("the enablement platform is not bound in this process")
+  return async (row, signal) => {
+    const file = await getFile(deps.files, scope, row.fileId)
+    if (file.size > MAX_AUTO_INDEX_BYTES)
+      throw Object.assign(new Error("the file is too large to read"), { permanent: "too_large" })
+    // The organization's eligibility and quota, as the enablement flow checks them, before the read is paid
+    // for: a rollout kill switch or a spent quota stops the next PDF (root's core-11 decision). The service
+    // still refuses past its quota at call time; this check only stops earlier.
+    const eligibility = await enablement.checkEligibility(scope)
+    if (!eligibility.ok)
+      throw Object.assign(new Error(eligibility.detail ?? eligibility.reason), { permanent: eligibility.reason })
+    if ((await enablement.indexingQuota(scope)) <= 0)
+      throw Object.assign(new Error("the indexing volume for today is spent"), { permanent: "quota_exhausted" })
+    signal?.throwIfAborted()
+    const bytes = await readFileBytes(deps, scope, row.fileId, MAX_AUTO_INDEX_BYTES)
+    await deps.beforeIndex?.(row.requestedBy)
+    signal?.throwIfAborted()
+    const read = await reader.read(bytes, {
+      filename: row.filename,
+      idempotencyKey: `knowledge.autoindex.${row.fileId}.${file.sha256}`,
+      ...(signal ? { signal } : {}),
+    })
+    const documentRef = await passages.reference(scope, row.fileId)
+    await recordPassages(deps, scope, row, documentRef, read)
+    await deps.beforeIndex?.(row.requestedBy)
+    signal?.throwIfAborted()
+    await passages.index(scope, {
+      collectionName: row.collectionName,
+      documentRef,
+      fileId: row.fileId,
+      passages: read,
+      ...(signal ? { signal } : {}),
+    })
+  }
+}
+
+/** The passages per page, comma-separated from page 1: what `passagesOf` turns back into references. */
+const pagesOf = (read: readonly KnowledgePassage[]): string => {
+  const counts: number[] = []
+  for (const passage of read) {
+    while (counts.length < passage.page) counts.push(0)
+    counts[passage.page - 1] = (counts[passage.page - 1] ?? 0) + 1
+  }
+  return counts.join(",")
+}
+
+const passagesOf = (pages: string): Array<{ page: number; index: number }> =>
+  pages === ""
+    ? []
+    : pages
+        .split(",")
+        .flatMap((count, at) => Array.from({ length: Number(count) }, (_, index) => ({ page: at + 1, index })))
+
+async function recordPassages(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  row: DocumentRow,
+  documentRef: string,
+  read: readonly KnowledgePassage[],
+): Promise<void> {
+  const fields = { documentRef, pages: pagesOf(read) }
+  await deps.data.unitOfWork.run(scope, async (tx) => {
+    const existing = (await tx.of(passageSets).list(passageSetOf(row))).items[0]
+    if (existing) await tx.of(passageSets).update(existing.id, fields, existing.version)
+    else await tx.of(passageSets).insert({ collectionName: row.collectionName, fileId: row.fileId, ...fields })
+  })
+}
+
+/** The query of a document's passage record: one per (collection, file), as the document row. */
+const passageSetOf = (row: { collectionName: string; fileId: string }) => ({
+  where: [
+    { field: "collectionName" as const, op: "eq" as const, value: row.collectionName },
+    { field: "fileId" as const, op: "eq" as const, value: row.fileId },
+  ],
+  limit: 1,
+})
+
+async function runIndex(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  documentId: string,
+  sendOf: () => Send,
+  options: { signal?: AbortSignal; finalAttempt?: boolean },
+): Promise<DocumentView> {
+  const row = await documentById(deps, scope, documentId)
+  if (row.status === "indexed" || row.status === "cancelled" || row.status === "failed") {
+    // A redelivered job or a crashed retry: the row says where the work stands.
+    return {
+      documentId: row.id,
+      collectionName: row.collectionName,
+      fileId: row.fileId,
+      filename: row.filename,
+      status: row.status,
+      error: row.error,
+      indexedAt: row.indexedAt,
+      requestedBy: row.requestedBy,
+    }
+  }
+  const send = sendOf()
+  const indexing = await deps.data
+    .forScope(scope)
+    .of(documents)
+    .update(row.id, { status: "indexing", error: null }, row.version)
+
+  try {
+    options.signal?.throwIfAborted()
+    if (!(await knowledgeEnabled(deps, scope)))
+      throw Object.assign(new Error("processing disabled"), { permanent: "feature_disabled" })
+    await deps.beforeIndex?.(row.requestedBy)
+    options.signal?.throwIfAborted()
+    await send(indexing, options.signal)
+    options.signal?.throwIfAborted()
+    const current = await deps.data.forScope(scope).of(documents).get(indexing.id)
+    if (!current || current.status === "cancelled") return documentView({ ...indexing, status: "cancelled" })
+    const done = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .update(indexing.id, { status: "indexed", indexedAt: new Date().toISOString(), error: null }, indexing.version)
+    return {
+      documentId: done.id,
+      collectionName: done.collectionName,
+      fileId: done.fileId,
+      filename: done.filename,
+      status: done.status as DocumentView["status"],
+      error: done.error,
+      indexedAt: done.indexedAt,
+      requestedBy: done.requestedBy,
+    }
+  } catch (error) {
+    if (options.signal?.aborted) throw error
+    const permanent = permanentFailure(error) ?? (options.finalAttempt === true ? "processing_exhausted" : null)
+    if (permanent === null) throw error
+    const current = await deps.data.forScope(scope).of(documents).get(indexing.id)
+    if (!current || current.status === "cancelled") return documentView({ ...indexing, status: "cancelled" })
+    const failed = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .update(indexing.id, { status: "failed", error: permanent }, indexing.version)
+    return {
+      documentId: failed.id,
+      collectionName: failed.collectionName,
+      fileId: failed.fileId,
+      filename: failed.filename,
+      status: "failed",
+      error: failed.error,
+      indexedAt: failed.indexedAt,
+      requestedBy: failed.requestedBy,
+    }
+  }
+}
+
+export async function removeDocument(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  documentId: string,
+  storageKey?: string,
+): Promise<void> {
+  const row = await deps.data.forScope(scope).of(documents).get(documentId)
+  if (!row) return
+  const set = (await deps.data.forScope(scope).of(passageSets).list(passageSetOf(row))).items[0]
+  if (set) {
+    if (!deps.passages) throw internal("the passage index is not bound in this process")
+    await deps.passages.remove(scope, { documentRef: set.documentRef, passages: passagesOf(set.pages) })
+    await deps.data.unitOfWork.run(scope, async (tx) => {
+      await tx.of(documents).delete(row.id)
+      await tx.of(passageSets).delete(set.id)
+    })
+    return
+  }
+  const service = deps.search
+  if (!service) throw internal("the knowledge search service is not bound in this process")
+  await service.remove(scope, {
+    collectionName: row.collectionName,
+    fileId: row.fileId,
+    ...(storageKey === undefined ? {} : { storageKey }),
+  })
+  await deps.data.forScope(scope).of(documents).delete(row.id)
+}
+
+async function readFileBytes(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  fileId: string,
+  maxBytes: number,
+): Promise<Uint8Array> {
+  const file = await getFile(deps.files, scope, fileId)
+  const response = await downloadFile(deps.files, scope, fileId)
+  if (!response.body) throw Object.assign(new Error("source missing"), { permanent: "not_indexable" })
+  const bytes = await readAll(response.body, maxBytes)
+  if (!file.sha256 || (await sha256Hex(bytes)) !== file.sha256)
+    throw Object.assign(new Error("source changed"), { permanent: "file_changed" })
+  return bytes
+}
+
+function decodeText(bytes: Uint8Array): string {
+  try {
+    return new TextDecoder("utf-8", { fatal: true }).decode(bytes)
+  } catch {
+    throw Object.assign(new Error("the file is not valid UTF-8 text"), { permanent: "not_text" })
+  }
+}
+
+/** Maps an indexing failure to the stable code recorded on the row; null when it is transient. */
+function permanentFailure(error: unknown): string | null {
+  const code = (error as { permanent?: unknown }).permanent
+  return (INDEX_FAILURES as readonly unknown[]).includes(code) ? (code as IndexFailure) : null
+}

modules/knowledge/core/ports.ts

@@ -0,0 +1,180 @@
+// The ports the knowledge module's use cases are written against, and the tokens the composition
+// roots bind. Core imports no SDK and no adapter (spec §2): the search service and the enablement
+// platform are shapes, and adapters/ is the only place an SDK exists.
+
+import { type Container, type TenantScope, token } from "@conusai-kit/core"
+import type { DataSource } from "@conusai-kit/data"
+import type { FilesDeps } from "@conusai-kit/files"
+import type { EnablementReason, KnowledgeRefusal, ServiceState } from "../contract/index.js"
+import { config, KNOWLEDGE_FLAG_DEFAULTS } from "../contract/index.js"
+
+/** A project scope: collections and documents live at this level. */
+export type KnowledgeScope = TenantScope & { workspaceId: string; projectId: string }
+
+/** An org scope: the enablement configuration is the organization's own. */
+export type KnowledgeOrgScope = TenantScope & { orgId: string }
+
+export interface KnowledgeDeps {
+  readonly data: DataSource
+  readonly files: FilesDeps
+  readonly flags: KnowledgeFlags
+  readonly beforeIndex?: (subjectUserId: string) => Promise<void>
+  /**
+   * The search service. Present for the effect procedure (search); a read or write procedure's deps
+   * never carry it, because resolving an effect port from either is a kind violation — the write
+   * only records the row and publishes the event, and the status read asks `KNOWLEDGE_READINESS`.
+   */
+  readonly search?: KnowledgeSearchService | undefined
+  /** The document reader and the passage index, for the automatic PDF intake (core-11); bound in the worker. */
+  readonly reader?: KnowledgeReader | undefined
+  readonly passages?: KnowledgePassageIndex | undefined
+  /** The organization's eligibility and indexing quota, which the PDF intake checks before each read. */
+  readonly enablement?: DocumentAiPlatform | undefined
+}
+
+/** One cited search answer. */
+export interface KnowledgeHit {
+  fileId: string
+  filename: string
+  collectionName: string
+  /** 0..1; higher is closer. */
+  score: number
+  chunk: number
+  snippet: string
+  page?: number
+  blockId?: string
+  chunkId?: string
+}
+
+/**
+ * The search service: index a document, search a collection with citations, remove a document, and
+ * report readiness. Its one adapter is the platform's managed documents-index service (ADR-061), which
+ * owns chunking, embeddings and the vector store. Every call leaves the process, so this port is an
+ * effect. `refusal` is why the service will not serve this deployment, when that is the answer.
+ */
+export interface KnowledgeSearchService {
+  readiness(scope: KnowledgeScope): Promise<{ state: ServiceState; reasons: string[]; refusal?: KnowledgeRefusal }>
+  index(
+    scope: KnowledgeScope,
+    input: { collectionName: string; fileId: string; filename: string; bytes: Uint8Array; signal?: AbortSignal },
+  ): Promise<void>
+  search(
+    scope: KnowledgeScope,
+    input: { collectionName: string; query: string; limit: number; actorId: string },
+  ): Promise<readonly KnowledgeHit[]>
+  /** `storageKey` names the object, resolved when the removal was asked for; only an older record lacks it, and then the file is looked up. */
+  remove(scope: KnowledgeScope, input: { collectionName: string; fileId: string; storageKey?: string }): Promise<void>
+}
+
+/**
+ * One passage of a read PDF, as the automatic intake indexes it: its page (1-based), its place on that
+ * page (0-based; with the page, its stable name across reads of the same document), the document's own
+ * block ids it holds lines of, and its text.
+ */
+export interface KnowledgePassage {
+  readonly page: number
+  readonly index: number
+  readonly blockIds: readonly string[]
+  readonly text: string
+}
+
+/**
+ * The platform's document reader, as the automatic intake uses it: a PDF in, its passages out. The
+ * same `idempotencyKey` reads the same document once. A document it can never make passages of throws
+ * with `permanent: "not_indexable"`; anything else is transient, and the queue retries the job.
+ */
+export interface KnowledgeReader {
+  read(
+    bytes: Uint8Array,
+    options: { filename: string; idempotencyKey: string; signal?: AbortSignal },
+  ): Promise<readonly KnowledgePassage[]>
+}
+
+/**
+ * The knowledge service's inline index, one item per passage, each under the file's reference plus its
+ * page and place. The reference is asked for first, so the caller can record what it is about to index
+ * before any of it is sent, and `remove` takes it back because the file may be gone by then: the service
+ * removes a reference only exactly, so removal names every passage that was indexed.
+ */
+export interface KnowledgePassageIndex {
+  reference(scope: KnowledgeScope, fileId: string): Promise<string>
+  index(
+    scope: KnowledgeScope,
+    input: {
+      collectionName: string
+      documentRef: string
+      fileId: string
+      passages: readonly KnowledgePassage[]
+      signal?: AbortSignal
+    },
+  ): Promise<void>
+  remove(
+    scope: KnowledgeScope,
+    input: { documentRef: string; passages: ReadonlyArray<Pick<KnowledgePassage, "page" | "index">> },
+  ): Promise<void>
+}
+
+/**
+ * What the status read asks of the deployment's search backend. It is a port of its own because a read
+ * handler may not resolve `KNOWLEDGE_SEARCH`, and must not build it: a factory runs with the resolving
+ * handler's restrictions, so a search service built for a read would hold a read-only data source.
+ */
+export type KnowledgeReadiness = Pick<KnowledgeSearchService, "readiness">
+
+/**
+ * The enablement platform: the checks the enablement flow needs, over the platform's own APIs.
+ * `checkEligibility` is where the plan capability and the rollout flags surface as typed reasons;
+ * `indexingQuota` answers today's remaining indexing volume.
+ */
+export interface DocumentAiPlatform {
+  checkEligibility(
+    scope: KnowledgeOrgScope,
+  ): Promise<{ ok: true } | { ok: false; reason: EnablementReason; detail?: string }>
+  indexingQuota(scope: KnowledgeOrgScope): Promise<number>
+}
+
+/** What a flag service answers for the project a call runs in. */
+export interface KnowledgeFlags {
+  enabled(flag: keyof typeof KNOWLEDGE_FLAG_DEFAULTS, scope: KnowledgeScope): boolean | Promise<boolean>
+}
+
+/**
+ * Bound by an app that evaluates flags (the platform's flag service). Unbound: knowledge is on for an
+ * organization once an administrator has enabled document AI, and off until then (`enablementFlags`).
+ */
+export const KNOWLEDGE_FLAGS = token<KnowledgeFlags>("knowledge.flags")
+
+/** The organization's own switch: the row `knowledge.settings.enableDocumentAi` writes. */
+export const enablementFlags = (data: DataSource): KnowledgeFlags => ({
+  enabled: async (flag, scope) => {
+    const row = (await data.forScope(scope).of(config).list({ limit: 1 })).items[0]
+    return row?.enabled ?? KNOWLEDGE_FLAG_DEFAULTS[flag]
+  },
+})
+
+export const flagsOf = (container: Container, data: DataSource): KnowledgeFlags =>
+  container.has(KNOWLEDGE_FLAGS) ? container.resolve(KNOWLEDGE_FLAGS) : enablementFlags(data)
+
+/** Bound by the worker and the web shell with the module's search backend adapter. */
+export const KNOWLEDGE_SEARCH = token<KnowledgeSearchService>("knowledge.search", { access: "effect" })
+
+/** Bound by the worker with the platform's document reader: the automatic PDF intake reads through it. */
+export const KNOWLEDGE_READER = token<KnowledgeReader>("knowledge.reader", { access: "effect" })
+
+/** Bound by the worker with the knowledge service's inline index: the PDF intake's passages, and their removal. */
+export const KNOWLEDGE_PASSAGES = token<KnowledgePassageIndex>("knowledge.passages", { access: "effect" })
+
+/** Bound by the web shell with the readiness of the service `KNOWLEDGE_SEARCH` serves. */
+export const KNOWLEDGE_READINESS = token<KnowledgeReadiness>("knowledge.readiness")
+
+/**
+ * Bound by the web shell (the enablement flow runs in a request) and by the worker (the PDF intake checks
+ * eligibility and quota again before each read, core-11).
+ */
+export const KNOWLEDGE_ENABLEMENT = token<DocumentAiPlatform>("knowledge.enablement", { access: "effect" })
+
+/** The per-run scope, under the module's own name (decision AT-F6). */
+export const KNOWLEDGE_SCOPE = token<TenantScope>("knowledge.scope")
+
+export const knowledgeEnabled = async (deps: Pick<KnowledgeDeps, "flags">, scope: KnowledgeScope): Promise<boolean> =>
+  deps.flags.enabled("knowledge.enabled", scope)

modules/knowledge/core/search.ts

@@ -0,0 +1,65 @@
+// The search: one collection, one query, cited answers. The query leaves the process (the procedure
+// is an effect), the ranking is the platform's service, and every answer carries where it came from:
+// the file, the chunk, and the snippet a person reads.
+
+import { AppError, forbidden, internal } from "@conusai-kit/core"
+import { getFile } from "@conusai-kit/files"
+import { documents, MAX_SEARCH_RESULTS, type SearchResult } from "../contract/index.js"
+import { requireCollection } from "./collections.js"
+import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+export async function searchKnowledge(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  input: { collectionName: string; query: string; limit?: number | undefined },
+): Promise<{ results: SearchResult[] }> {
+  if (!(await knowledgeEnabled(deps, scope))) throw forbidden("knowledge is switched off for this project")
+  await requireCollection(deps, scope, input.collectionName)
+  const service = deps.search
+  if (!service) throw internal("the knowledge search service is not bound in this process")
+  const limit = Math.min(input.limit ?? 10, MAX_SEARCH_RESULTS)
+  const hits = await service.search(scope, {
+    collectionName: input.collectionName,
+    query: input.query,
+    limit,
+    actorId: scope.actor.userId,
+  })
+  const results: SearchResult[] = []
+  for (const hit of hits) {
+    // The current scoped source of truth revokes a citation before the remote deletion job finishes.
+    const current = (
+      await deps.data
+        .forScope(scope)
+        .of(documents)
+        .list({
+          where: [
+            { field: "collectionName", op: "eq", value: input.collectionName },
+            { field: "fileId", op: "eq", value: hit.fileId },
+            { field: "status", op: "eq", value: "indexed" },
+          ],
+          limit: 1,
+        })
+    ).items[0]
+    if (!current || hit.collectionName !== input.collectionName) continue
+    // The original file can be deleted before its outbox cleanup is delivered.
+    // Current scoped file visibility is mandatory, independent of stale index rows.
+    try {
+      await getFile(deps.files, scope, hit.fileId)
+    } catch (error) {
+      if (error instanceof AppError && error.status === 404) continue
+      throw error
+    }
+    results.push({
+      fileId: hit.fileId,
+      filename: hit.filename,
+      collectionName: hit.collectionName,
+      snippet: hit.snippet,
+      score: hit.score,
+      chunk: hit.chunk,
+      ...(hit.page === undefined ? {} : { page: hit.page }),
+      ...(hit.blockId === undefined ? {} : { blockId: hit.blockId }),
+      ...(hit.chunkId === undefined ? {} : { chunkId: hit.chunkId }),
+    })
+  }
+  return { results }
+}

modules/knowledge/core/settings.ts

@@ -0,0 +1,108 @@
+// The enablement flow: one idempotent procedure pair an organization administrator runs, with the
+// platform's checks in front of any configuration change and every denial a typed reason code the
+// UI maps to an upgrade or policy state.
+//
+//   enableDocumentAi: already enabled -> answers the current configuration, writes nothing.
+//     Otherwise: eligibility (plan capability + rollout flags) -> quota -> one transaction persists
+//     the configuration. Nothing is provisioned: retrieval is the platform's service (ADR-061), and an
+//     app keeps no vector store. A failure at any step writes nothing and is retirable: the same call
+//     again resumes from the first failing check.
+//
+//   disableDocumentAi: preserves everything — collections, documents, the service's index and the
+//     recorded configuration stay; only `enabled` flips, so re-enabling resumes without duplication.
+
+import { AppError } from "@conusai-kit/core"
+import { config, type EnablementReason, type KnowledgeBackend } from "../contract/index.js"
+import type { DocumentAiPlatform, KnowledgeOrgScope } from "./ports.js"
+
+/** A typed enablement denial: the problem's code is the stable reason the UI maps to a state. */
+export function enablementDenied(reason: EnablementReason, detail?: string): AppError {
+  const status = reason === "quota_exhausted" ? 429 : 403
+  const title =
+    reason === "plan_required"
+      ? "The organization's plan does not include document AI"
+      : reason === "feature_disabled"
+        ? "Document AI is disabled by policy"
+        : "The indexing quota is exhausted"
+  return new AppError({
+    type: `/problems/${reason}`,
+    title,
+    status,
+    code: reason,
+    ...(detail === undefined ? {} : { detail }),
+  })
+}
+
+export interface SettingsView {
+  enabled: boolean
+  /** Who serves this organization once it has a configuration: only the platform does. */
+  backend: KnowledgeBackend | null
+}
+
+async function currentConfig(deps: { data: import("@conusai-kit/data").DataSource }, scope: KnowledgeOrgScope) {
+  const row = (await deps.data.forScope(scope).of(config).list({ limit: 1 })).items[0]
+  return row ?? null
+}
+
+export async function getSettings(
+  deps: { data: import("@conusai-kit/data").DataSource },
+  scope: KnowledgeOrgScope,
+): Promise<SettingsView> {
+  const row = await currentConfig(deps, scope)
+  return { enabled: row?.enabled ?? false, backend: row ? "platform" : null }
+}
+
+export interface SettingsDeps {
+  data: import("@conusai-kit/data").DataSource
+  platform: DocumentAiPlatform
+  now?: () => number
+}
+
+export async function enableDocumentAi(deps: SettingsDeps, scope: KnowledgeOrgScope): Promise<SettingsView> {
+  const existing = await currentConfig(deps, scope)
+  if (existing?.enabled) return { enabled: true, backend: "platform" }
+
+  const eligibility = await deps.platform.checkEligibility(scope)
+  if (!eligibility.ok) throw enablementDenied(eligibility.reason, eligibility.detail)
+
+  const remaining = await deps.platform.indexingQuota(scope)
+  if (remaining <= 0) throw enablementDenied("quota_exhausted", "the indexing volume for today is spent")
+
+  await deps.data.unitOfWork.run(scope, async (tx) => {
+    const row = (await tx.of(config).list({ limit: 1 })).items[0]
+    const patch = {
+      enabled: true,
+      backend: "platform" as const,
+      source: "managed" as const,
+      serviceId: null,
+      url: null,
+      updatedBy: scope.actor.userId,
+      configuredAt: new Date((deps.now ?? Date.now)()).toISOString(),
+    }
+    if (row) await tx.of(config).update(row.id, patch, row.version)
+    else await tx.of(config).insert(patch)
+  })
+  return { enabled: true, backend: "platform" }
+}
+
+export async function disableDocumentAi(
+  deps: Pick<SettingsDeps, "data" | "now">,
+  scope: KnowledgeOrgScope,
+): Promise<{ enabled: false }> {
+  await deps.data.unitOfWork.run(scope, async (tx) => {
+    const row = (await tx.of(config).list({ limit: 1 })).items[0]
+    if (!row?.enabled) return
+    // Everything else is preserved on purpose: the recorded configuration, every collection and
+    // document, and the service's index. Disabling fences new work; it deletes nothing.
+    await tx.of(config).update(
+      row.id,
+      {
+        enabled: false,
+        updatedBy: scope.actor.userId,
+        configuredAt: new Date((deps.now ?? Date.now)()).toISOString(),
+      },
+      row.version,
+    )
+  })
+  return { enabled: false }
+}

modules/knowledge/core/status.ts

@@ -0,0 +1,49 @@
+// The status a page reads: the module flag, the search service's readiness (separate from the
+// indexing indicator, which is the per-document rows), and the document counts by progress state.
+
+import { type DocumentStatus, documents, type KnowledgeRefusal } from "../contract/index.js"
+import { type KnowledgeDeps, type KnowledgeReadiness, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+const PAGE = 200
+
+export interface KnowledgeStatus {
+  enabled: boolean
+  service: {
+    state: "registered" | "indexing" | "ready" | "degraded" | "disabled" | "unavailable"
+    reasons: string[]
+    refusal?: KnowledgeRefusal
+  }
+  documents: Record<DocumentStatus, number>
+}
+
+/** What the status read needs; `readiness` is absent in a process that binds no search backend. */
+export type KnowledgeStatusDeps = Pick<KnowledgeDeps, "data" | "flags"> & {
+  readonly readiness?: KnowledgeReadiness | undefined
+}
+
+export async function knowledgeStatusOf(deps: KnowledgeStatusDeps, scope: KnowledgeScope): Promise<KnowledgeStatus> {
+  const counts: Record<DocumentStatus, number> = { queued: 0, indexing: 0, indexed: 0, failed: 0, cancelled: 0 }
+  let cursor: string | undefined
+  for (;;) {
+    const page = await deps.data
+      .forScope(scope)
+      .of(documents)
+      .list({ limit: PAGE, ...(cursor === undefined ? {} : { cursor }) })
+    for (const row of page.items) counts[row.status as DocumentStatus] += 1
+    if (!page.hasMore) break
+    cursor = page.cursor
+  }
+  const enabled = await knowledgeEnabled(deps, scope)
+  const readiness =
+    deps.readiness === undefined
+      ? { state: "unavailable" as const, reasons: ["the knowledge search service is not bound in this process"] }
+      : await deps.readiness.readiness(scope).catch((error: unknown) => ({
+          state: "unavailable" as const,
+          reasons: [
+            error instanceof Error
+              ? (error.message.split("\n")[0] ?? error.message)
+              : "the search service did not answer",
+          ],
+        }))
+  return { enabled, service: readiness, documents: counts }
+}