ConusAI vendor v0.1.0-beta.114
@@ -0,0 +1,280 @@
|
||||
// The managed search service: the platform's documents-index service through the kit-platform
|
||||
// Knowledge client. Indexing and search run server-side there (the service owns chunking,
|
||||
// embeddings and the vector store); this module supplies the job lifecycle and the citations it
|
||||
// answers with.
|
||||
//
|
||||
// Contract notes (the service's real shapes): the index submission carries the content's SHA-256
|
||||
// (computed here from the job's bytes) as its version assertion; a quota denial surfaces on the
|
||||
// job, which the poll below reads as the job's typed error. Blob pathnames and returned citations
|
||||
// resolve through the files module's scoped rows, so an opaque file id never becomes a pathname
|
||||
// and a foreign or stale service reference never becomes a valid app hit.
|
||||
|
||||
import { createHash } from "node:crypto"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import {
|
||||
type FilesDeps,
|
||||
findFileStorageByKey,
|
||||
getFileStorage,
|
||||
platformBlobPathname,
|
||||
platformBlobStorageKey,
|
||||
} from "@conusai-kit/files"
|
||||
import { KNOWLEDGE_LIMITS, type Knowledge, type KnowledgeClient, KnowledgeError } from "@conusai-kit/platform"
|
||||
import type { KnowledgeHit, KnowledgePassageIndex, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js"
|
||||
import { platformKnowledgeReadiness } from "./readiness.js"
|
||||
import { permanentRefusal } from "./refusals.js"
|
||||
|
||||
/** The service's tenant partition of a project: a search of the project and its indexed documents agree on it. */
|
||||
const tenantOf = (scope: KnowledgeScope): string =>
|
||||
createHash("sha256")
|
||||
.update(JSON.stringify([scope.orgId, scope.workspaceId, scope.projectId]))
|
||||
.digest("hex")
|
||||
|
||||
/** A passage's reference: the file's own plus `#p{page}.{index}` (the automatic PDF intake, core-11). */
|
||||
const passageRef = (documentRef: string, passage: { page: number; index: number }): string =>
|
||||
`${documentRef}#p${passage.page}.${passage.index}`
|
||||
|
||||
const PASSAGE_REF = /^(.+)#p([1-9][0-9]*)\.(0|[1-9][0-9]*)$/
|
||||
|
||||
/**
|
||||
* `environment` is the deployment's (`CONUSAI_ENVIRONMENT`): the service keeps a collection in one environment,
|
||||
* and a deployment token reaches only its own.
|
||||
*/
|
||||
export function platformKnowledgeSearch(
|
||||
client: Knowledge,
|
||||
files: FilesDeps,
|
||||
options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {},
|
||||
): KnowledgeSearchService {
|
||||
const environment = options.environment?.trim()
|
||||
const configuredEnvironment = (): string => {
|
||||
if (!environment)
|
||||
throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing")
|
||||
return environment
|
||||
}
|
||||
const unwrap = (error: unknown): never => {
|
||||
// The client's errors are already typed by reason; a non-knowledge failure is unavailable.
|
||||
if (error instanceof KnowledgeError) throw error
|
||||
throw new KnowledgeError(
|
||||
"runtime_unavailable",
|
||||
error instanceof Error ? error.message : "the knowledge service failed",
|
||||
true,
|
||||
)
|
||||
}
|
||||
/** The same, for an index job: a refusal waiting cannot fix is recorded on the document, not retried. */
|
||||
const unwrapJob = (error: unknown): never => {
|
||||
if (error instanceof KnowledgeError) {
|
||||
const permanent = permanentRefusal(error)
|
||||
if (permanent !== undefined) throw Object.assign(error, { permanent })
|
||||
}
|
||||
return unwrap(error)
|
||||
}
|
||||
const documentRef = async (scope: KnowledgeScope, fileId: string): Promise<string> => {
|
||||
const reference = await getFileStorage(files, scope, fileId)
|
||||
return platformBlobPathname(scope, reference.storageKey)
|
||||
}
|
||||
|
||||
return {
|
||||
readiness(scope) {
|
||||
if (!environment)
|
||||
return Promise.resolve({
|
||||
state: "unavailable",
|
||||
reasons: ["CONUSAI_ENVIRONMENT is not set"],
|
||||
refusal: "configuration_missing",
|
||||
})
|
||||
return platformKnowledgeReadiness(client).readiness(scope)
|
||||
},
|
||||
|
||||
async index(scope, input) {
|
||||
try {
|
||||
// The service answers 404 for a collection it does not hold. An app's collection is only a row of
|
||||
// this module (a write cannot leave the process), so the first index creates it; creating it again
|
||||
// finds it.
|
||||
const activeEnvironment = configuredEnvironment()
|
||||
const reference = await documentRef(scope, input.fileId)
|
||||
const signal = AbortSignal.any([
|
||||
AbortSignal.timeout(options.timeoutMs ?? 300_000),
|
||||
...(input.signal ? [input.signal] : []),
|
||||
])
|
||||
signal.throwIfAborted()
|
||||
await client.createCollection({ collection: input.collectionName, environment: activeEnvironment }, { signal })
|
||||
const { jobId } = await client.index(
|
||||
{
|
||||
collection: input.collectionName,
|
||||
documentRef: reference,
|
||||
options: {
|
||||
visibility: "tenant",
|
||||
tenant: tenantOf(scope),
|
||||
contentSha256: createHash("sha256").update(input.bytes).digest("hex"),
|
||||
},
|
||||
},
|
||||
{ signal },
|
||||
)
|
||||
// The service's job is durable; poll until it leaves the queue. A caller-side cancel is
|
||||
// the job's own AbortSignal, checked between polls.
|
||||
for (;;) {
|
||||
signal.throwIfAborted()
|
||||
const job = await client.job(jobId, { signal })
|
||||
if (job.status === "succeeded") return
|
||||
if (job.status === "failed")
|
||||
throw new KnowledgeError(
|
||||
job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable",
|
||||
job.error ?? "the indexing job failed",
|
||||
)
|
||||
if (job.status === "cancelled")
|
||||
throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled")
|
||||
await delay(options.intervalMs ?? 1_000, undefined, { signal })
|
||||
}
|
||||
} catch (error) {
|
||||
return unwrapJob(error)
|
||||
}
|
||||
},
|
||||
|
||||
async search(scope, input) {
|
||||
try {
|
||||
configuredEnvironment()
|
||||
const results = await client.search({
|
||||
collection: input.collectionName,
|
||||
query: input.query,
|
||||
limit: input.limit,
|
||||
tenant: tenantOf(scope),
|
||||
})
|
||||
const hits: KnowledgeHit[] = []
|
||||
for (const [at, hit] of results.entries()) {
|
||||
// A passage of a PDF the intake indexed names its file's reference, and cites from its metadata:
|
||||
// the service reports pages only for sources it parsed itself.
|
||||
const passage = PASSAGE_REF.exec(hit.documentRef)
|
||||
const storageKey = platformBlobStorageKey(scope, passage?.[1] ?? hit.documentRef)
|
||||
if (storageKey === null) continue
|
||||
const reference = await findFileStorageByKey(files, scope, storageKey)
|
||||
if (reference === null) continue
|
||||
const metadata = "metadata" in hit ? (hit.metadata as Readonly<Record<string, string>>) : {}
|
||||
if (passage && metadata.file_id !== reference.file.fileId) continue
|
||||
const page = passage ? Number(passage[2]) : hit.citations[0]?.page
|
||||
const blockId = passage ? metadata.block_ids?.split(",")[0] || undefined : hit.citations[0]?.blockId
|
||||
hits.push({
|
||||
fileId: reference.file.fileId,
|
||||
filename: reference.file.filename,
|
||||
collectionName: input.collectionName,
|
||||
score: hit.score,
|
||||
chunk: at + 1,
|
||||
snippet: hit.snippet,
|
||||
...(page === undefined ? {} : { page }),
|
||||
...(blockId === undefined ? {} : { blockId }),
|
||||
...("chunkId" in hit && typeof hit.chunkId === "string" ? { chunkId: hit.chunkId } : {}),
|
||||
})
|
||||
}
|
||||
return hits
|
||||
} catch (error) {
|
||||
// Nothing has been indexed into this collection yet, so the service has not heard of it.
|
||||
if (error instanceof KnowledgeError && error.reason === "unavailable_resource") return []
|
||||
return unwrap(error)
|
||||
}
|
||||
},
|
||||
|
||||
async remove(scope, input) {
|
||||
try {
|
||||
// Every removal asked for since F407 names its object; only a record from before it is looked up.
|
||||
const reference =
|
||||
input.storageKey === undefined
|
||||
? await documentRef(scope, input.fileId)
|
||||
: platformBlobPathname(scope, input.storageKey)
|
||||
await client.remove(reference)
|
||||
} catch (error) {
|
||||
return unwrap(error)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The automatic PDF intake's index (core-11): one inline item per passage, in the project's tenant partition,
|
||||
* under the file's Blob reference plus `#p{page}.{index}`, with its page, its blocks and its file in metadata.
|
||||
* Items go at most the service's count per call, and every item's durable job is waited for; a refusal waiting
|
||||
* cannot fix is the document's, as with a whole file.
|
||||
*/
|
||||
export function platformPassageIndex(
|
||||
client: Pick<KnowledgeClient, "createCollection" | "indexInline" | "job" | "remove">,
|
||||
files: FilesDeps,
|
||||
options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {},
|
||||
): KnowledgePassageIndex {
|
||||
const environment = options.environment?.trim()
|
||||
const failure = (error: unknown): never => {
|
||||
if (error instanceof KnowledgeError) {
|
||||
const permanent = permanentRefusal(error)
|
||||
if (permanent !== undefined) throw Object.assign(error, { permanent })
|
||||
throw error
|
||||
}
|
||||
throw new KnowledgeError(
|
||||
"runtime_unavailable",
|
||||
error instanceof Error ? error.message : "the knowledge service failed",
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
async reference(scope, fileId) {
|
||||
const stored = await getFileStorage(files, scope, fileId)
|
||||
return platformBlobPathname(scope, stored.storageKey)
|
||||
},
|
||||
|
||||
async index(scope, input) {
|
||||
try {
|
||||
if (!environment)
|
||||
throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing")
|
||||
const signal = AbortSignal.any([
|
||||
AbortSignal.timeout(options.timeoutMs ?? 300_000),
|
||||
...(input.signal ? [input.signal] : []),
|
||||
])
|
||||
await client.createCollection({ collection: input.collectionName, environment }, { signal })
|
||||
const tenant = tenantOf(scope)
|
||||
const jobs: string[] = []
|
||||
for (let at = 0; at < input.passages.length; at += KNOWLEDGE_LIMITS.inlineItems) {
|
||||
signal.throwIfAborted()
|
||||
const accepted = await client.indexInline({
|
||||
collection: input.collectionName,
|
||||
items: input.passages.slice(at, at + KNOWLEDGE_LIMITS.inlineItems).map((passage) => ({
|
||||
documentRef: passageRef(input.documentRef, passage),
|
||||
text: passage.text,
|
||||
visibility: "tenant" as const,
|
||||
tenant,
|
||||
metadata: {
|
||||
page: String(passage.page),
|
||||
...(passage.blockIds.length === 0 ? {} : { block_ids: passage.blockIds.join(",") }),
|
||||
file_id: input.fileId,
|
||||
},
|
||||
})),
|
||||
})
|
||||
jobs.push(...accepted.map((job) => job.jobId))
|
||||
}
|
||||
for (const jobId of jobs) {
|
||||
for (;;) {
|
||||
signal.throwIfAborted()
|
||||
const job = await client.job(jobId, { signal })
|
||||
if (job.status === "succeeded") break
|
||||
if (job.status === "failed")
|
||||
throw new KnowledgeError(
|
||||
job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable",
|
||||
job.error ?? "the indexing job failed",
|
||||
)
|
||||
if (job.status === "cancelled")
|
||||
throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled")
|
||||
await delay(options.intervalMs ?? 1_000, undefined, { signal })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
return failure(error)
|
||||
}
|
||||
},
|
||||
|
||||
async remove(_scope, input) {
|
||||
for (const passage of input.passages) {
|
||||
try {
|
||||
await client.remove(passageRef(input.documentRef, passage))
|
||||
} catch (error) {
|
||||
// A passage the service never indexed (the run was refused before it) is already removed.
|
||||
if (error instanceof KnowledgeError && error.reason === "unavailable_resource") continue
|
||||
failure(error)
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user