// The documents: request an index (a web write that records the row and publishes the outbox // event the worker's subscriber turns into the index job), request a removal the same way, and // list a collection's documents with their progress states. import { AppError, conflict, forbidden, notFound, validationFailed } from "@conusai-kit/core" import { getFile, getFileStorage } from "@conusai-kit/files" import { type DocumentView, documentIndexRequested, documentRemoveRequested, documents, INDEXABLE_CONTENT_TYPES, MAX_INDEX_BYTES, } from "../contract/index.js" import { requireCollection } from "./collections.js" import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js" const PAGE = 200 export const documentView = (row: { id: string collectionName: string fileId: string filename: string status: DocumentView["status"] error: string | null indexedAt: string | null requestedBy: string }): DocumentView => ({ documentId: row.id, collectionName: row.collectionName, fileId: row.fileId, filename: row.filename, status: row.status, error: row.error, indexedAt: row.indexedAt, requestedBy: row.requestedBy, }) export async function listDocuments( deps: KnowledgeDeps, scope: KnowledgeScope, collectionName: string, ): Promise<{ documents: DocumentView[] }> { const found = [] let cursor: string | undefined for (;;) { const page = await deps.data .forScope(scope) .of(documents) .list({ where: [{ field: "collectionName", op: "eq", value: collectionName }], order: [{ field: "createdAt", direction: "desc" }], limit: PAGE, ...(cursor === undefined ? {} : { cursor }), }) found.push(...page.items) if (!page.hasMore) break cursor = page.cursor } return { documents: found.map((row) => documentView(row)) } } /** The most ids one batch-get reads (contract C-05). */ export const MAX_BATCH = 100 /** * The live collection's read-back: the documents with these ids, in whichever collection of the project. * A row this caller cannot see is not in the answer: the repository's tenant wall leaves it out. */ export async function getManyDocuments( deps: KnowledgeDeps, scope: KnowledgeScope, documentIds: readonly string[], ): Promise { const page = await deps.data .forScope(scope) .of(documents) .list({ where: [{ field: "id", op: "in", values: [...new Set(documentIds)] }], limit: MAX_BATCH }) return page.items.map((row) => documentView(row)) } /** * Asks for a document's indexing. The write is three things in one transaction: the durable row * (the UI's progress states and the job's idempotency key), the outbox event the worker's * subscriber enqueues the index job from, and nothing else — the call to the platform's knowledge * service happens in the worker, never in a request (AGENTS.md: long work is a job). * * Idempotent by (collection, file): asking again for a pair that is queued or indexing answers * the same row and publishes nothing; asking for an indexed or failed pair re-queues the one row, * so a retry can never duplicate a document. */ export async function requestIndex( deps: KnowledgeDeps, scope: KnowledgeScope, input: { collectionName: string; fileId: string }, ): Promise { // The tenant-wall reads come before the flag check: another tenant's ids answer NotFound, never // a policy refusal, so ids do not leak existence behind a disabled flag. await requireCollection(deps, scope, input.collectionName) const file = await getFile(deps.files, scope, input.fileId) if (!(INDEXABLE_CONTENT_TYPES as readonly string[]).includes(file.contentType)) { throw validationFailed( [{ path: "fileId", message: `${file.contentType} cannot be indexed; index a text file` }], "this file type is not indexable", ) } if (file.size > MAX_INDEX_BYTES) { throw validationFailed( [{ path: "fileId", message: `the file is larger than ${MAX_INDEX_BYTES} bytes` }], "the file is too large to index", ) } if (!(await knowledgeEnabled(deps, scope))) throw forbidden("knowledge is switched off for this project") return deps.data.unitOfWork.run(scope, async (tx) => { const existing = ( await tx.of(documents).list({ where: [ { field: "collectionName", op: "eq", value: input.collectionName }, { field: "fileId", op: "eq", value: input.fileId }, ], limit: 1, }) ).items[0] if (existing && (existing.status === "queued" || existing.status === "indexing")) { return documentView(existing) } const row = existing !== undefined ? await tx .of(documents) .update( existing.id, { status: "queued", error: null, indexedAt: null, requestedBy: scope.actor.userId }, existing.version, ) : await tx.of(documents).insert({ collectionName: input.collectionName, fileId: input.fileId, filename: file.filename, status: "queued", error: null, indexedAt: null, requestedBy: scope.actor.userId, }) await tx.publish(documentIndexRequested, { orgId: scope.orgId, workspaceId: scope.workspaceId, projectId: scope.projectId, documentId: row.id, collectionName: input.collectionName, fileId: input.fileId, }) return documentView(row) }) } /** Asks for a document's removal: the row and the event in one transaction, the job in the worker. */ export async function requestRemove( deps: KnowledgeDeps, scope: KnowledgeScope, input: { collectionName: string; fileId: string; storageKey?: string }, ): Promise { const current = ( await deps.data .forScope(scope) .of(documents) .list({ where: [ { field: "collectionName", op: "eq", value: input.collectionName }, { field: "fileId", op: "eq", value: input.fileId }, ], limit: 1, }) ).items[0] // The tenant-wall read comes first, as in requestIndex: another tenant's ids answer NotFound. if (!current) throw notFound("no such document") if (current.status === "cancelled") return documentView(current) // Every removal names the object it removes, resolved when it is asked for, not when the job runs: by then the // file may be gone (F407). A deletion brings the key it had; a person's removal reads it from the stored file. const storageKey = input.storageKey ?? (await storageKeyOf(deps, scope, input.fileId)) return deps.data.unitOfWork.run(scope, async (tx) => { const row = ( await tx.of(documents).list({ where: [ { field: "collectionName", op: "eq", value: input.collectionName }, { field: "fileId", op: "eq", value: input.fileId }, ], limit: 1, }) ).items[0] if (!row) throw notFound("no such document") if (row.status === "cancelled") return documentView(row) const next = await tx.of(documents).update(row.id, { status: "cancelled", error: null }, row.version) await tx.publish(documentRemoveRequested, { orgId: scope.orgId, workspaceId: scope.workspaceId, projectId: scope.projectId, documentId: row.id, collectionName: input.collectionName, fileId: input.fileId, storageKey, }) return documentView(next) }) } /** The document row a job resumes by; NotFound when another tenant's id was named. */ export async function documentById(deps: KnowledgeDeps, scope: KnowledgeScope, documentId: string) { const row = await deps.data.forScope(scope).of(documents).get(documentId) if (!row) throw notFound("no such document") return row } /** * The stored file's object key. A file on its way out (being deleted, or gone) is refused rather than removed * unkeyed: its deletion publishes `FileDeleted` with the key, and that removal finds the document still active. */ async function storageKeyOf(deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string): Promise { try { return (await getFileStorage(deps.files, scope, fileId)).storageKey } catch (error) { if (error instanceof AppError && error.status === 404) { throw conflict("the file is being deleted; its indexed text is removed with it") } throw error } } /** A source deletion revokes every collection's citation and durably requests cleanup. * Does not require processing enablement: deleting retained data is independent of inference. * `storageKey` is the key the deleted file had (`FileDeleted`): its row is gone, so the removal cannot look it up. */ export async function requestSourceRemoval( deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string, storageKey: string, ): Promise { let cursor: string | undefined for (;;) { const page = await deps.data .forScope(scope) .of(documents) .list({ where: [{ field: "fileId", op: "eq", value: fileId }], limit: PAGE, ...(cursor === undefined ? {} : { cursor }), }) for (const row of page.items) await requestRemove(deps, scope, { collectionName: row.collectionName, fileId, storageKey }) if (!page.hasMore) break cursor = page.cursor } }