ConusAI vendor v0.1.0-beta.114

modules/knowledge/contract/collections.ts

@@ -0,0 +1,20 @@
+// The live collection of the knowledge module (umbrella TS6.1). The live server announces a change of
+// `knowledge_document` -- an index asked for, a job taking it, the platform answering indexed or failed, a
+// removal -- as `{ entity: "knowledge.document", id }`, ids only; a reader reads the document back through
+// `knowledge.document.getMany`, so the procedure's permission and the repository's tenant wall decide what a
+// member sees. A topic is a project, so a page that shows one collection filters the rows by its name.
+// `pnpm kit check` refuses a collection without its batch-get procedure.
+
+import type { ModuleCollection } from "@conusai-kit/core"
+
+export const DOCUMENT_COLLECTION = {
+  entity: "knowledge.document",
+  table: "knowledge_document",
+  key: "documentId",
+  scopeLevel: "project",
+  read: "knowledge.knowledge.search",
+  list: "knowledge.document.list",
+  getMany: "knowledge.document.getMany",
+} as const satisfies ModuleCollection
+
+export const KNOWLEDGE_COLLECTIONS: readonly ModuleCollection[] = [DOCUMENT_COLLECTION]

modules/knowledge/contract/constants.ts

@@ -0,0 +1,50 @@
+// The constants of the knowledge module: its feature flag, jobs, events, audit actions and the
+// indexable content types.
+
+/** The module flag: knowledge retrieval and indexing, off until an administrator enables document AI. */
+export const KNOWLEDGE_FLAG = "knowledge.enabled"
+
+export const KNOWLEDGE_FLAG_DEFAULTS = {
+  [KNOWLEDGE_FLAG]: false,
+} as const
+
+/** The worker job that hands one document to the platform's knowledge service. */
+export const INDEX_JOB = "knowledge.index"
+
+/** The worker job that removes one document from the platform's knowledge service. */
+export const REMOVE_JOB = "knowledge.remove"
+
+export const COLLECTION_CREATED = "knowledge.collection.created"
+export const DOCUMENT_INDEX_REQUESTED = "knowledge.document.index_requested"
+export const DOCUMENT_REMOVE_REQUESTED = "knowledge.document.remove_requested"
+export const DOCUMENT_AI_ENABLED = "knowledge.settings.document_ai_enabled"
+export const DOCUMENT_AI_DISABLED = "knowledge.settings.document_ai_disabled"
+/** A project's automatic PDF indexing switched on (naming a collection) or off. */
+export const AUTO_INDEX_CHANGED = "knowledge.autoindex.changed"
+
+/** The worker job that reads one uploaded PDF and indexes its passages into the project's opted-in collection. */
+export const AUTO_INDEX_JOB = "knowledge.autoindex"
+
+/** The files processor that enqueues it on every upload of a matching type; also its jobs' singleton namespace. */
+export const AUTO_INDEX_PROCESSOR_KEY = "knowledge.autoindex"
+
+/** What the automatic indexing reads, through the platform's document reader. */
+export const AUTO_INDEX_CONTENT_TYPES = ["application/pdf"] as const
+
+/** A PDF's ceiling for the reader: the files module's upload ceiling, as the documents module reads one. */
+export const MAX_AUTO_INDEX_BYTES = 50 * 1024 * 1024
+
+/**
+ * What the indexer reads as text. PDFs and scans are the documents module's worked example; this
+ * module indexes text-shaped files, and refuses anything else at the door with a named field.
+ */
+export const INDEXABLE_CONTENT_TYPES = ["text/plain", "text/markdown", "application/json", "text/csv"] as const
+
+/** A search query's ceiling; a retrieval question is a sentence, not a document. */
+export const MAX_QUERY_LENGTH = 500
+
+/** What one search returns at most, however much the caller asks (the retrieval limit contract). */
+export const MAX_SEARCH_RESULTS = 25
+
+/** An indexed file's ceiling: one megabyte of text is already far past useful chunking. */
+export const MAX_INDEX_BYTES = 1_048_576

modules/knowledge/contract/entities.ts

@@ -0,0 +1,105 @@
+// The tables of the knowledge module. Five.
+//
+// `knowledge_collection` is one named collection of documents, project-owned; the collection name
+// is the slug procedures take (a procedure's input carries slugs, never ids). `knowledge_document`
+// is one file's indexing state in one collection — the row the UI's progress states read, and the
+// idempotency key of the index job (a file is indexed into a collection once, and re-indexing the
+// same pair resumes the one row). `knowledge_config` is the organization's enablement state: the
+// durable answer to "is document AI on" and which backend serves it. `knowledge_auto_index` is a
+// project's opt-in to indexing its uploaded PDFs, and `knowledge_passage_set` what one such PDF sent
+// to the service, so removing it can tombstone each passage. No secret ever lands here.
+
+import { defineEntity, idSchema, instantSchema } from "@conusai-kit/data"
+import { z } from "zod"
+import { DOCUMENT_STATUSES, KNOWLEDGE_BACKENDS } from "./statuses.js"
+
+export const collections = defineEntity({
+  name: "collection",
+  table: "knowledge_collection",
+  tenancy: "project",
+  fields: {
+    name: z.string().min(1).max(100),
+    createdBy: z.string().min(1).max(128),
+  },
+  unique: [["name"]],
+})
+
+export const documents = defineEntity({
+  name: "document",
+  table: "knowledge_document",
+  tenancy: "project",
+  fields: {
+    collectionName: z.string().min(1).max(100),
+    /** The row of the files module this document indexes: an opaque id, never a foreign key. */
+    fileId: idSchema,
+    filename: z.string().min(1).max(255),
+    status: z.enum(DOCUMENT_STATUSES),
+    /** A stable failure code when the indexing failed; null otherwise. */
+    error: z.string().max(200).nullable(),
+    indexedAt: z.string().nullable(),
+    requestedBy: z.string().min(1).max(128),
+  },
+  // One row per (collection, file): a re-index resumes the same row instead of duplicating it.
+  unique: [["collectionName", "fileId"]],
+  indexes: [["collectionName"]],
+})
+
+export const config = defineEntity({
+  name: "config",
+  table: "knowledge_config",
+  tenancy: "org",
+  fields: {
+    enabled: z.boolean(),
+    /** Which search backend serves this organization. */
+    backend: z.enum(KNOWLEDGE_BACKENDS),
+    // The three below held the app's own vector store, which is gone (ADR-061); the migration that made
+    // them has run, so they stay as "managed, nothing to link" and nothing reads them.
+    source: z.enum(["managed"]),
+    serviceId: z.string().max(128).nullable(),
+    url: z.string().max(500).nullable(),
+    updatedBy: z.string().min(1).max(128),
+    /** When the configuration last changed; the managed updated_at is the row's own bookkeeping. */
+    configuredAt: instantSchema,
+  },
+})
+
+/**
+ * A project's opt-in to indexing its uploaded PDFs automatically (core-11): no row means off. It names
+ * the collection the files go into and the administrator who turned it on, whose permission every
+ * indexing job checks again, since the job runs for nobody else.
+ */
+export const autoIndex = defineEntity({
+  name: "autoindex",
+  table: "knowledge_auto_index",
+  tenancy: "project",
+  fields: {
+    collectionName: z.string().min(1).max(100),
+    enabledBy: z.string().min(1).max(128),
+    enabledAt: instantSchema,
+  },
+})
+
+/**
+ * The passages an automatic PDF index sent for one document (core-11). The service indexed one inline
+ * item per passage under the file's reference plus `#p{page}.{index}`, and removes a reference only
+ * exactly, so the row keeps the file's reference (which outlives the file) and the count of passages
+ * on each page, comma-separated from page 1 ("2,0,1": two on page 1, none on page 2, one on page 3).
+ */
+export const passageSets = defineEntity({
+  name: "passageset",
+  table: "knowledge_passage_set",
+  tenancy: "project",
+  fields: {
+    collectionName: z.string().min(1).max(100),
+    fileId: idSchema,
+    documentRef: z.string().min(1).max(1024),
+    pages: z.string().max(65_536),
+  },
+  unique: [["collectionName", "fileId"]],
+})
+
+/** The tables of the first migration, as first shipped. */
+export const KNOWLEDGE_ENTITIES_V1 = Object.freeze([collections, documents, config])
+
+/** Every table this module owns, in the order its migrations create them. */
+export const KNOWLEDGE_ENTITIES = Object.freeze([...KNOWLEDGE_ENTITIES_V1, autoIndex, passageSets])

modules/knowledge/contract/events.ts

@@ -0,0 +1,41 @@
+// The events of the knowledge module. A write procedure publishes the request; the module's own
+// subscriber (running in the worker's outbox relay) enqueues the job, which is how a web request
+// never waits on the platform's knowledge service.
+
+import { defineEvent } from "@conusai-kit/core"
+import { z } from "zod"
+
+const inProject = {
+  orgId: z.string(),
+  workspaceId: z.string(),
+  projectId: z.string(),
+}
+
+/** A document was asked to be indexed: the subscriber enqueues the index job. */
+export const documentIndexRequested = defineEvent(
+  "knowledge.documentIndexRequested",
+  z.object({
+    ...inProject,
+    documentId: z.string(),
+    collectionName: z.string(),
+    fileId: z.string(),
+  }),
+)
+
+/**
+ * A document was asked to be removed: the subscriber enqueues the remove job. `storageKey` names the object to
+ * remove, resolved when the removal is asked for (from `FileDeleted`, or from the stored file a person removes),
+ * because the file may be gone when the job runs. It is optional only for a record written before it existed.
+ */
+export const documentRemoveRequested = defineEvent(
+  "knowledge.documentRemoveRequested",
+  z.object({
+    ...inProject,
+    documentId: z.string(),
+    collectionName: z.string(),
+    fileId: z.string(),
+    storageKey: z.string().optional(),
+  }),
+)
+
+export const KNOWLEDGE_EVENTS = [documentIndexRequested, documentRemoveRequested]

modules/knowledge/contract/index.ts

@@ -0,0 +1,66 @@
+// The public API of the knowledge module: what a page, another module and the OpenAPI document may
+// import. Nothing here reaches core/, adapters/ or a driver.
+
+export { DOCUMENT_COLLECTION, KNOWLEDGE_COLLECTIONS } from "./collections.js"
+export {
+  AUTO_INDEX_CHANGED,
+  AUTO_INDEX_CONTENT_TYPES,
+  AUTO_INDEX_JOB,
+  AUTO_INDEX_PROCESSOR_KEY,
+  COLLECTION_CREATED,
+  DOCUMENT_AI_DISABLED,
+  DOCUMENT_AI_ENABLED,
+  DOCUMENT_INDEX_REQUESTED,
+  DOCUMENT_REMOVE_REQUESTED,
+  INDEX_JOB,
+  INDEXABLE_CONTENT_TYPES,
+  KNOWLEDGE_FLAG,
+  KNOWLEDGE_FLAG_DEFAULTS,
+  MAX_AUTO_INDEX_BYTES,
+  MAX_INDEX_BYTES,
+  MAX_QUERY_LENGTH,
+  MAX_SEARCH_RESULTS,
+  REMOVE_JOB,
+} from "./constants.js"
+export {
+  autoIndex,
+  collections,
+  config,
+  documents,
+  KNOWLEDGE_ENTITIES,
+  KNOWLEDGE_ENTITIES_V1,
+  passageSets,
+} from "./entities.js"
+export { documentIndexRequested, documentRemoveRequested, KNOWLEDGE_EVENTS } from "./events.js"
+export type { CollectionView, DocumentView, SearchResult } from "./procedures.js"
+export {
+  autoIndexGet,
+  autoIndexSet,
+  collectionCreate,
+  collectionList,
+  collectionName,
+  collectionViewSchema,
+  documentGetMany,
+  documentIndex,
+  documentList,
+  documentRemove,
+  documentViewSchema,
+  KNOWLEDGE_PROCEDURES,
+  knowledgeSearch,
+  knowledgeStatus,
+  searchResultSchema,
+  settingsDisableDocumentAi,
+  settingsEnableDocumentAi,
+  settingsGet,
+} from "./procedures.js"
+export type { DocumentStatus, EnablementReason, IndexFailure } from "./statuses.js"
+export {
+  ENABLEMENT_REASONS,
+  INDEX_FAILURES,
+  KNOWLEDGE_BACKENDS,
+  KNOWLEDGE_REFUSALS,
+  type KnowledgeBackend,
+  type KnowledgeRefusal,
+  SERVICE_STATES,
+  type ServiceState,
+} from "./statuses.js"

modules/knowledge/contract/procedures.ts

@@ -0,0 +1,229 @@
+// The procedure contracts of the knowledge module: schemas and names, no handler and no port.
+//
+// The module is a retrieval layer over the app's documents: a collection of text files is indexed
+// (in the worker: the platform's knowledge service reads, chunks and embeds it), searched
+// with cited snippets, and managed through an explicit, idempotent enablement flow. The enablement
+// procedure is named enableDocumentAi per the Document AI enablement contract; its permission is the module's
+// manage permission, so the admin gate is one line in the manifest.
+
+import { procedure } from "@conusai-kit/core"
+import { z } from "zod"
+import { AUTO_INDEX_CHANGED, COLLECTION_CREATED, DOCUMENT_AI_DISABLED, DOCUMENT_AI_ENABLED } from "./constants.js"
+import { type DocumentStatus, KNOWLEDGE_BACKENDS, KNOWLEDGE_REFUSALS, type ServiceState } from "./statuses.js"
+
+const inProject = { org: z.string(), workspace: z.string(), project: z.string() }
+const inOrg = { org: z.string() }
+
+export const collectionName = z
+  .string()
+  .min(1)
+  .max(100)
+  .regex(/^[a-z][a-z0-9-]*$/, "lower-case letters, digits and dashes")
+
+export const collectionViewSchema = z.object({
+  name: collectionName,
+  documentCount: z.number().int(),
+  indexedCount: z.number().int(),
+  createdBy: z.string(),
+})
+export type CollectionView = z.output<typeof collectionViewSchema>
+
+export const documentViewSchema = z.object({
+  documentId: z.string(),
+  collectionName: collectionName,
+  fileId: z.string(),
+  filename: z.string(),
+  status: z.enum(["queued", "indexing", "indexed", "failed", "cancelled"]),
+  error: z.string().nullable(),
+  indexedAt: z.string().nullable(),
+  requestedBy: z.string(),
+})
+export type DocumentView = z.output<typeof documentViewSchema>
+
+export const searchResultSchema = z.object({
+  fileId: z.string(),
+  filename: z.string(),
+  collectionName: collectionName,
+  snippet: z.string(),
+  page: z.number().int().min(1).optional(),
+  blockId: z.string().optional(),
+  chunkId: z.string().optional(),
+  /** 0..1; higher is closer. */
+  score: z.number(),
+  /** Result ordinal only. Page/block/chunkId carry actual returned source locators when available. */
+  chunk: z.number().int().min(1),
+})
+export type SearchResult = z.output<typeof searchResultSchema>
+
+export const collectionList = procedure("knowledge.collection.list")
+  .kind("read")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .input(z.object({ ...inProject }))
+  .output(z.object({ collections: z.array(collectionViewSchema) }))
+  .contract()
+
+export const collectionCreate = procedure("knowledge.collection.create")
+  .kind("write")
+  .scope("project")
+  .permission("knowledge.collection.manage")
+  .input(z.object({ ...inProject, name: collectionName }))
+  .output(collectionViewSchema)
+  .audit(COLLECTION_CREATED)
+  .contract()
+
+export const documentIndex = procedure("knowledge.document.index")
+  .kind("write")
+  .scope("project")
+  .permission("knowledge.knowledge.manage")
+  .input(z.object({ ...inProject, collectionName: collectionName, fileId: z.string() }))
+  .output(documentViewSchema)
+  .contract()
+
+export const documentList = procedure("knowledge.document.list")
+  .kind("read")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .input(z.object({ ...inProject, collectionName: collectionName }))
+  .output(z.object({ documents: z.array(documentViewSchema) }))
+  .contract()
+
+/**
+ * The live collection's batch-get (contract C-05): the documents with these ids, in whichever collection of
+ * the project, under the list's permission. An id of another tenant, of another project or of nothing is
+ * left out of the answer.
+ */
+export const documentGetMany = procedure("knowledge.document.getMany")
+  .kind("read")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .input(z.object({ ...inProject, ids: z.array(z.string().min(1).max(64)).min(1).max(100) }))
+  .output(z.object({ items: z.array(documentViewSchema) }))
+  .contract()
+
+export const documentRemove = procedure("knowledge.document.remove")
+  .kind("write")
+  .scope("project")
+  .permission("knowledge.knowledge.manage")
+  .input(z.object({ ...inProject, collectionName: collectionName, fileId: z.string() }))
+  .output(documentViewSchema)
+  .contract()
+
+/**
+ * The retrieval call. It embeds the query through the AI gateway before it searches, so it is an
+ * effect: a read procedure cannot leave the process. Exposed as an MCP tool on purpose (`.tool()`).
+ */
+export const knowledgeSearch = procedure("knowledge.search")
+  .kind("effect")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .tool()
+  .input(
+    z.object({
+      ...inProject,
+      collectionName: collectionName,
+      query: z.string().min(1).max(500),
+      limit: z.number().int().min(1).max(25).optional(),
+    }),
+  )
+  .output(z.object({ results: z.array(searchResultSchema) }))
+  .contract()
+
+export const knowledgeStatus = procedure("knowledge.status")
+  .kind("read")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .input(z.object({ ...inProject }))
+  .output(
+    z.object({
+      /** The module flag's answer for this project. */
+      enabled: z.boolean(),
+      /** The service readiness, kept separate from the indexing indicator below. */
+      service: z.object({
+        state: z.enum(["registered", "indexing", "ready", "degraded", "disabled", "unavailable"]),
+        reasons: z.array(z.string()),
+        /** Why the service will not serve this deployment, when that is the answer: the page's own state. */
+        refusal: z.enum(KNOWLEDGE_REFUSALS).optional(),
+      }),
+      documents: z.object({
+        queued: z.number().int(),
+        indexing: z.number().int(),
+        indexed: z.number().int(),
+        failed: z.number().int(),
+        cancelled: z.number().int(),
+      }),
+    }),
+  )
+  .contract()
+
+export const settingsGet = procedure("knowledge.settings.get")
+  .kind("read")
+  .scope("org")
+  .permission("knowledge.settings.manage")
+  .input(z.object({ ...inOrg }))
+  .output(
+    z.object({
+      enabled: z.boolean(),
+      backend: z.enum(KNOWLEDGE_BACKENDS).nullable(),
+    }),
+  )
+  .contract()
+
+/** Its checks and the managed link call the platform's APIs, so it is an effect, like the search. */
+export const settingsEnableDocumentAi = procedure("knowledge.settings.enableDocumentAi")
+  .kind("effect")
+  .scope("org")
+  .permission("knowledge.settings.manage")
+  .input(z.object({ ...inOrg }))
+  .output(z.object({ enabled: z.boolean(), backend: z.enum(KNOWLEDGE_BACKENDS) }))
+  .audit(DOCUMENT_AI_ENABLED)
+  .contract()
+
+export const settingsDisableDocumentAi = procedure("knowledge.settings.disableDocumentAi")
+  .kind("write")
+  .scope("org")
+  .permission("knowledge.settings.manage")
+  .input(z.object({ ...inOrg }))
+  .output(z.object({ enabled: z.boolean() }))
+  .audit(DOCUMENT_AI_DISABLED)
+  .contract()
+
+/** Whether this project indexes its uploaded PDFs automatically, and into which collection. */
+export const autoIndexGet = procedure("knowledge.autoindex.get")
+  .kind("read")
+  .scope("project")
+  .permission("knowledge.knowledge.search")
+  .input(z.object({ ...inProject }))
+  .output(z.object({ collectionName: collectionName.nullable() }))
+  .contract()
+
+/**
+ * Turns automatic PDF indexing on (into an existing collection) or off (`null`). The administrator who
+ * turns it on is the one every indexing job re-checks: their permission is the job's authority.
+ */
+export const autoIndexSet = procedure("knowledge.autoindex.set")
+  .kind("write")
+  .scope("project")
+  .permission("knowledge.knowledge.manage")
+  .input(z.object({ ...inProject, collectionName: collectionName.nullable() }))
+  .output(z.object({ collectionName: collectionName.nullable() }))
+  .audit(AUTO_INDEX_CHANGED)
+  .contract()
+
+export const KNOWLEDGE_PROCEDURES = [
+  collectionList,
+  collectionCreate,
+  documentIndex,
+  documentList,
+  documentGetMany,
+  documentRemove,
+  knowledgeSearch,
+  knowledgeStatus,
+  settingsGet,
+  settingsEnableDocumentAi,
+  settingsDisableDocumentAi,
+  autoIndexGet,
+  autoIndexSet,
+]
+
+export type { DocumentStatus, ServiceState }

modules/knowledge/contract/statuses.ts

@@ -0,0 +1,53 @@
+// The statuses a knowledge document moves through. `unavailable` and `disabled` are not rows: they
+// are the module's and the service's answers, and the UI maps all eight states from what a page
+// reads (the module flag, the service readiness, the document row).
+
+export const DOCUMENT_STATUSES = ["queued", "indexing", "indexed", "failed", "cancelled"] as const
+export type DocumentStatus = (typeof DOCUMENT_STATUSES)[number]
+
+/** The search backend a deployment binds: the platform's managed service, and no other (ADR-061). */
+export const KNOWLEDGE_BACKENDS = ["platform"] as const
+export type KnowledgeBackend = (typeof KNOWLEDGE_BACKENDS)[number]
+
+/** The readiness states the search port answers (the managed toolset's readiness + unavailable). */
+export const SERVICE_STATES = ["registered", "indexing", "ready", "degraded", "disabled", "unavailable"] as const
+export type ServiceState = (typeof SERVICE_STATES)[number]
+
+/** The enablement flow's typed refusal reasons, surfaced in the UI as states. */
+export const ENABLEMENT_REASONS = ["plan_required", "feature_disabled", "quota_exhausted"] as const
+export type EnablementReason = (typeof ENABLEMENT_REASONS)[number]
+
+/**
+ * Why the platform's knowledge service will not serve this deployment, as the pages show it. The service's
+ * reason codes reach the module as these (adapters/refusals.ts): `scope_missing` is the service refusing
+ * the deployment's token, whose `conusai.json` asks for `knowledge.collections:use`.
+ */
+export const KNOWLEDGE_REFUSALS = [
+  "plan_required",
+  "feature_disabled",
+  "quota_exhausted",
+  "budget_exhausted",
+  "scope_missing",
+  "indexing",
+  "approval_required",
+  "runtime_unavailable",
+  "configuration_missing",
+] as const
+export type KnowledgeRefusal = (typeof KNOWLEDGE_REFUSALS)[number]
+
+/** The indexing failure codes a document row records: the file's own, and a refusal waiting cannot fix. */
+export const INDEX_FAILURES = [
+  "processing_exhausted",
+  "configuration_missing",
+  "not_indexable",
+  "too_large",
+  "not_text",
+  "file_changed",
+  "plan_required",
+  "feature_disabled",
+  "quota_exhausted",
+  "budget_exhausted",
+  "scope_missing",
+  "approval_required",
+] as const
+export type IndexFailure = (typeof INDEX_FAILURES)[number]