@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "@app/worker",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "The background process of this app: module jobs, module cron, the outbox relay, and the migrate job that owns the schema.",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.build.json",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"start": "node dist/main.js",
|
||||
"seed": "node dist/seed-main.js",
|
||||
"migrate": "node dist/migrate.js",
|
||||
"clean": "rm -rf dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@app/config": "workspace:*",
|
||||
"@app/modules": "workspace:*",
|
||||
"@conusai-kit/auth": "catalog:kit",
|
||||
"@conusai-kit/auth-convex-better-auth": "catalog:kit",
|
||||
"@conusai-kit/core": "catalog:kit",
|
||||
"@conusai-kit/data": "catalog:kit",
|
||||
"@conusai-kit/data-kysely": "catalog:kit",
|
||||
"@conusai-kit/files": "catalog:kit",
|
||||
"@conusai-kit/jobs-pgboss": "catalog:kit",
|
||||
"@conusai-kit/platform": "catalog:kit",
|
||||
"@conusai-kit/proposals": "catalog:kit",
|
||||
"@conusai-kit/tenancy": "catalog:kit",
|
||||
"@conusai-sdk/node-sdk": "catalog:",
|
||||
"@conusai-sdk/observe-node": "catalog:",
|
||||
"kysely": "catalog:",
|
||||
"pg": "catalog:",
|
||||
"pg-boss": "catalog:",
|
||||
"zod": "catalog:",
|
||||
"@conusai-kit/agent": "catalog:kit",
|
||||
"@conusai-kit/guides": "catalog:kit",
|
||||
"@conusai-kit/support": "catalog:kit",
|
||||
"@conusai-kit/jobs-bullmq": "catalog:kit",
|
||||
"@conusai-kit/data-mongo": "catalog:kit",
|
||||
"@conusai-kit/live-server": "catalog:kit",
|
||||
"ioredis": "catalog:",
|
||||
"@conusai-kit/host": "catalog:kit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "26.6.5",
|
||||
"@types/pg": "catalog:",
|
||||
"typescript": "5.9.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The worker's part of the Convex Better Auth adapter (AUTH_PROVIDER=convex-better-auth): an account
|
||||
// deletion records the provider whose account it ends, and the sweep that resumes one ends the Better
|
||||
// Auth account at that step, as the web server would (apps/web/src/server/better-auth.ts).
|
||||
// `kit template:prune --auth oidc` removes this file with the lines that call it.
|
||||
|
||||
import { ACCOUNT_IDENTITY_PROVIDER } from "@app/modules/account"
|
||||
import { betterAuthAccountEndFromEnv } from "@conusai-kit/auth-convex-better-auth/account"
|
||||
import type { Container } from "@conusai-kit/core"
|
||||
|
||||
/**
|
||||
* Binds the account's end when Better Auth is the app's provider. Read at start-up, so a worker that
|
||||
* lacks the deployment's variables stops by name instead of holding every deletion at that step.
|
||||
*/
|
||||
export function bindBetterAuthAccountEnd(container: Container, env: Record<string, string | undefined>): void {
|
||||
if (env.AUTH_PROVIDER?.trim() !== "convex-better-auth") return
|
||||
container.value(ACCOUNT_IDENTITY_PROVIDER, betterAuthAccountEndFromEnv(env))
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// What the worker and the migrate job need of one dialect profile's engine. Each engine is a folder
|
||||
// of its own (`mariadb/`, `mongodb/`), so `kit template:prune --profile` removes the other one whole.
|
||||
|
||||
import type { DialectProfile } from "@app/config"
|
||||
import type { Registry } from "@conusai-kit/core"
|
||||
import type { AnyEntityDef, DataSource, SystemDataSource } from "@conusai-kit/data"
|
||||
|
||||
export type DialectData = DataSource & SystemDataSource & { close(): Promise<void> }
|
||||
|
||||
/** A migration of either dialect, by the name the runner records. */
|
||||
export interface NamedMigration {
|
||||
readonly name: string
|
||||
}
|
||||
|
||||
export interface DialectEngine {
|
||||
readonly profile: DialectProfile
|
||||
/** The profile's data source over its one URL (MongoDB: refuses a standalone server, AT-D15). */
|
||||
connect(env: Readonly<Record<string, string | undefined>>, entities: readonly AnyEntityDef[]): Promise<DialectData>
|
||||
/** The kernel's files, the job adapter's table and every module's files, in the runner's form. */
|
||||
migrations(registry: Registry, root?: string): Promise<NamedMigration[]>
|
||||
/** Applies them on the profile's one URL. */
|
||||
migrate(
|
||||
env: Readonly<Record<string, string | undefined>>,
|
||||
registry: Registry,
|
||||
): Promise<{ applied: string[]; skipped: string[] }>
|
||||
/** Deletes the live change log past retention, on an engine that has one (mariadb). */
|
||||
pruneLive?(env: Readonly<Record<string, string | undefined>>, hours: number): Promise<void>
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The mariadb engine of the worker and the migrate job (profiles/mariadb/README.md): the Kysely
|
||||
// MySQL dialect over MARIADB_URL, the kernel's and the modules' SQL files, the job adapter's table
|
||||
// generated from its entity as the Postgres profile's 0001_kernel.sql carries it, and the live
|
||||
// change log with one row-trigger capture per live collection (TS6.2).
|
||||
|
||||
import { readFile } from "node:fs/promises"
|
||||
import {
|
||||
DIALECT_JOBS_MIGRATION,
|
||||
DIALECT_LIVE_MIGRATION,
|
||||
dialectMigrationFiles,
|
||||
dialectModuleMigrationFiles,
|
||||
liveCaptureMigrations,
|
||||
} from "@app/config"
|
||||
import type { Registry } from "@conusai-kit/core"
|
||||
import {
|
||||
connectMysqlDataSourceFromEnv,
|
||||
type Migration,
|
||||
mysqlPool,
|
||||
mysqlSchemaStatements,
|
||||
runMysqlMigrationsFromEnv,
|
||||
} from "@conusai-kit/data-kysely/mysql"
|
||||
import { KIT_JOBS_ENTITIES } from "@conusai-kit/jobs-pgboss"
|
||||
import { attachMysqlCaptureStatements, liveMysqlSchemaStatements, pruneMysqlLiveLog } from "@conusai-kit/live-server"
|
||||
import { splitStatements } from "../../migrations.js"
|
||||
import type { DialectEngine } from "../engine.js"
|
||||
|
||||
/** `kit_processed_events`, the outbox relay's exactly-once ledger. */
|
||||
export function mariadbJobsMigration(): Migration {
|
||||
const kernel = mysqlSchemaStatements([]).length
|
||||
return { name: DIALECT_JOBS_MIGRATION, statements: mysqlSchemaStatements(KIT_JOBS_ENTITIES).slice(kernel) }
|
||||
}
|
||||
|
||||
/** The live change log, then each live collection's row triggers after its table. */
|
||||
export function mariadbLiveMigrations(registry: Registry): Migration[] {
|
||||
return [
|
||||
{ name: DIALECT_LIVE_MIGRATION, statements: liveMysqlSchemaStatements() },
|
||||
...liveCaptureMigrations(registry, "mariadb").map(({ name, entity, table, scopeLevel }) => ({
|
||||
name,
|
||||
statements: attachMysqlCaptureStatements({ entity, table, scopeLevel }),
|
||||
})),
|
||||
]
|
||||
}
|
||||
|
||||
async function migrations(registry: Registry, root?: string): Promise<Migration[]> {
|
||||
const files = [
|
||||
...(await dialectMigrationFiles("mariadb", root)),
|
||||
...(await dialectModuleMigrationFiles(registry, "mariadb", root)),
|
||||
]
|
||||
const read = await Promise.all(
|
||||
files.map(async (file) => ({ name: file.name, statements: splitStatements(await readFile(file.path, "utf8")) })),
|
||||
)
|
||||
return [...read, mariadbJobsMigration(), ...mariadbLiveMigrations(registry)]
|
||||
}
|
||||
|
||||
export const mariadbEngine: DialectEngine = {
|
||||
profile: "mariadb",
|
||||
connect: async (env, entities) => connectMysqlDataSourceFromEnv(env, { entities }),
|
||||
migrations,
|
||||
migrate: async (env, registry) => runMysqlMigrationsFromEnv(env, await migrations(registry)),
|
||||
async pruneLive(env, hours) {
|
||||
const pool = mysqlPool(env.MARIADB_URL ?? "", { connectionLimit: 1 })
|
||||
try {
|
||||
await pruneMysqlLiveLog(pool.promise(), hours)
|
||||
} finally {
|
||||
await pool.promise().end()
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The mongodb engine of the worker and the migrate job (profiles/mongodb/README.md): the official
|
||||
// driver over MONGODB_URL, which refuses a standalone server (AT-D15), the kernel's and the modules'
|
||||
// JSON command files, the job adapter's collection generated from its entity, and the pre-images
|
||||
// each live collection's deletes need on the change stream (TS6.2).
|
||||
|
||||
import { readFile } from "node:fs/promises"
|
||||
import {
|
||||
DIALECT_JOBS_MIGRATION,
|
||||
dialectMigrationFiles,
|
||||
dialectModuleMigrationFiles,
|
||||
liveCaptureMigrations,
|
||||
} from "@app/config"
|
||||
import type { Registry } from "@conusai-kit/core"
|
||||
import {
|
||||
connectMongoDataSourceFromEnv,
|
||||
MONGO_KERNEL_COMMANDS,
|
||||
type MongoCommand,
|
||||
type MongoMigration,
|
||||
mongoCommandsMigration,
|
||||
mongoSchemaCommands,
|
||||
runMongoMigrationsFromEnv,
|
||||
} from "@conusai-kit/data-mongo"
|
||||
import { KIT_JOBS_ENTITIES } from "@conusai-kit/jobs-pgboss"
|
||||
import { mongoLivePreImageCommand } from "@conusai-kit/live-server"
|
||||
import type { DialectEngine } from "../engine.js"
|
||||
|
||||
/** One per live collection: keep the document before a change, so a delete names its organization. */
|
||||
export function mongodbLiveMigrations(registry: Registry): MongoMigration[] {
|
||||
return liveCaptureMigrations(registry, "mongodb").map(({ name, table }) => {
|
||||
const command = mongoLivePreImageCommand({ table })
|
||||
return {
|
||||
name,
|
||||
source: JSON.stringify(command),
|
||||
up: async (db) => {
|
||||
await db.command(command)
|
||||
},
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** `kit_processed_events`, the outbox relay's exactly-once ledger. */
|
||||
export function mongodbJobsCommands(): MongoCommand[] {
|
||||
return mongoSchemaCommands(KIT_JOBS_ENTITIES).slice(MONGO_KERNEL_COMMANDS.length)
|
||||
}
|
||||
|
||||
async function migrations(registry: Registry, root?: string): Promise<MongoMigration[]> {
|
||||
const files = [
|
||||
...(await dialectMigrationFiles("mongodb", root)),
|
||||
...(await dialectModuleMigrationFiles(registry, "mongodb", root)),
|
||||
]
|
||||
const read = await Promise.all(
|
||||
files.map(async (file) =>
|
||||
mongoCommandsMigration(file.name, JSON.parse(await readFile(file.path, "utf8")) as MongoCommand[]),
|
||||
),
|
||||
)
|
||||
return [
|
||||
...read,
|
||||
mongoCommandsMigration(DIALECT_JOBS_MIGRATION, mongodbJobsCommands()),
|
||||
...mongodbLiveMigrations(registry),
|
||||
]
|
||||
}
|
||||
|
||||
export const mongodbEngine: DialectEngine = {
|
||||
profile: "mongodb",
|
||||
connect: (env, entities) => connectMongoDataSourceFromEnv(env, { entities }),
|
||||
migrations,
|
||||
migrate: async (env, registry) => runMongoMigrationsFromEnv(env, await migrations(registry)),
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The worker and the migrate job of the mariadb and mongodb profiles (spec §5.1;
|
||||
// profiles/<name>/README.md). One login and no row-level security (AT-D20 is Postgres-only), the
|
||||
// data source of the profile's engine, BullMQ on the linked Valkey for module jobs, and the kit's
|
||||
// outbox relay over the same data source, so on these two profiles, as on postgres, a domain event
|
||||
// reaches each subscriber at least once and its writes through the delivery's own transaction
|
||||
// (`JOB_REPOS`) land exactly once. The convex profile's relay is at least once only.
|
||||
//
|
||||
// `kit template:prune --profile` keeps this folder for mariadb and mongodb (and the one engine
|
||||
// folder of the chosen profile); postgres and convex remove it with every `kit:profile-dialect` line.
|
||||
|
||||
import { writeFile } from "node:fs/promises"
|
||||
import {
|
||||
type AppEnv,
|
||||
businessServiceComposition,
|
||||
type DialectProfile,
|
||||
type PlatformClients,
|
||||
platformClients,
|
||||
profileRegistry,
|
||||
type ServerObservability,
|
||||
} from "@app/config"
|
||||
import { appModules } from "@app/modules"
|
||||
import { Container, type Registry } from "@conusai-kit/core"
|
||||
import { DATA_SOURCE, entitiesOf, IDEMPOTENCY, JOB_QUEUE, OUTBOX, SYSTEM_DATA } from "@conusai-kit/data"
|
||||
import {
|
||||
type BullmqJobDef,
|
||||
type BullmqWorker,
|
||||
createBullmqJobQueue,
|
||||
createBullmqWorker,
|
||||
} from "@conusai-kit/jobs-bullmq"
|
||||
import { createOutboxRelay, KIT_JOBS_ENTITIES, type Relay } from "@conusai-kit/jobs-pgboss"
|
||||
import { createTenancyHooks, TENANCY } from "@conusai-kit/tenancy"
|
||||
import { Redis } from "ioredis"
|
||||
import { logWorkerEvent, traceJobs } from "../instrument.js"
|
||||
import { bindModulePorts, nativeHelpSelectors } from "../ports.js"
|
||||
import type { DialectData, DialectEngine, NamedMigration } from "./engine.js"
|
||||
import { mariadbEngine } from "./mariadb/index.js" // kit:profile-mariadb
|
||||
import { mongodbEngine } from "./mongodb/index.js" // kit:profile-mongodb
|
||||
import { dialectSchedules } from "./schedules.js"
|
||||
|
||||
/** The engines this app was created with: one after `kit template:prune --profile`. */
|
||||
const ENGINES: Partial<Record<DialectProfile, DialectEngine>> = {
|
||||
mariadb: mariadbEngine, // kit:profile-mariadb
|
||||
mongodb: mongodbEngine, // kit:profile-mongodb
|
||||
}
|
||||
|
||||
export interface DialectWorkerRuntime {
|
||||
readonly profile: DialectProfile
|
||||
readonly registry: Registry
|
||||
readonly data: DialectData
|
||||
readonly container: Container
|
||||
readonly platform: PlatformClients
|
||||
readonly worker: BullmqWorker
|
||||
readonly relay: Relay
|
||||
close(): Promise<void>
|
||||
}
|
||||
|
||||
/** The BullMQ prefix of this app (`kit-jobs-<slug>`): the web shell's `queueSlug`, from the same project id. */
|
||||
export function queueSlug(env: Pick<AppEnv, "deployment">): string {
|
||||
const id = env.deployment.CONUSAI_PROJECT_ID.toLowerCase().replace(/[^a-z0-9-]+/g, "-")
|
||||
return `p-${id}`.replace(/-+$/, "")
|
||||
}
|
||||
|
||||
/** The engine this environment was loaded for; refused by name when the app was pruned without it. */
|
||||
export function engineOf(env: AppEnv): DialectEngine {
|
||||
const profile: DialectProfile | null = env.mariadb ? "mariadb" : env.mongodb ? "mongodb" : null
|
||||
if (!profile) throw new Error("the selected profile has no MariaDB or MongoDB worker")
|
||||
const engine = ENGINES[profile]
|
||||
if (!engine) throw new Error(`this app was created without the ${profile} profile`)
|
||||
return engine
|
||||
}
|
||||
|
||||
/** The engine of a profile, for the migrate job and the tests. */
|
||||
export function engineFor(profile: DialectProfile): DialectEngine {
|
||||
const engine = ENGINES[profile]
|
||||
if (!engine) throw new Error(`this app was created without the ${profile} profile`)
|
||||
return engine
|
||||
}
|
||||
|
||||
/** The registry's jobs as the BullMQ worker takes them: the same payload schema and handler. */
|
||||
export function bullmqJobs(registry: Registry): Record<string, BullmqJobDef> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(registry.jobs).map(([name, { def }]) => [
|
||||
name,
|
||||
{ payload: def.payload, handler: (payload, ctx) => def.handler(payload, ctx) },
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
/** Every migration a dialect profile applies, in the runner's form. */
|
||||
export function dialectMigrations(
|
||||
registry: Registry,
|
||||
profile: DialectProfile,
|
||||
root?: string,
|
||||
): Promise<NamedMigration[]> {
|
||||
return engineFor(profile).migrations(registry, root)
|
||||
}
|
||||
|
||||
export async function buildDialectWorkerRuntime(
|
||||
env: AppEnv,
|
||||
observe: ServerObservability,
|
||||
): Promise<DialectWorkerRuntime> {
|
||||
const engine = engineOf(env)
|
||||
const redisUrl = env.redis?.url
|
||||
if (!redisUrl) throw new Error(`the ${engine.profile} profile has no REDIS_URL`)
|
||||
const registry = profileRegistry(traceJobs(appModules, observe), engine.profile)
|
||||
const platform = platformClients(env, registry)
|
||||
const data = await engine.connect(process.env, [...entitiesOf(registry, engine.profile), ...KIT_JOBS_ENTITIES])
|
||||
|
||||
const connection = new Redis(redisUrl, { maxRetriesPerRequest: null })
|
||||
const slug = queueSlug(env)
|
||||
const queue = createBullmqJobQueue({ connection, slug })
|
||||
|
||||
const container = new Container()
|
||||
container.value(DATA_SOURCE, data)
|
||||
container.value(SYSTEM_DATA, data)
|
||||
container.value(OUTBOX, data.outbox)
|
||||
container.value(IDEMPOTENCY, data.idempotency)
|
||||
container.value(JOB_QUEUE, queue)
|
||||
container.register(TENANCY, () => createTenancyHooks({ data, registry }), { lifetime: "transient" })
|
||||
bindModulePorts(
|
||||
container,
|
||||
platform,
|
||||
registry,
|
||||
businessServiceComposition(env, appModules, engine.profile, { model: platform.model, ...nativeHelpSelectors })
|
||||
.serviceReadiness,
|
||||
)
|
||||
|
||||
const log = (event: { kind: string; [field: string]: unknown }) => logWorkerEvent(observe, event)
|
||||
const worker = createBullmqWorker({
|
||||
connection,
|
||||
slug,
|
||||
container,
|
||||
dataSource: data,
|
||||
system: data,
|
||||
jobs: bullmqJobs(registry),
|
||||
log,
|
||||
// The module cron of this profile (dialect/schedules.ts), installed on the queue at start.
|
||||
registry,
|
||||
schedules: dialectSchedules(registry, engine.profile),
|
||||
})
|
||||
const relay = createOutboxRelay({ registry, dataSource: data, container, log })
|
||||
|
||||
return {
|
||||
profile: engine.profile,
|
||||
registry,
|
||||
data,
|
||||
container,
|
||||
platform,
|
||||
worker,
|
||||
relay,
|
||||
close: async () => {
|
||||
await relay.stop().catch(() => undefined)
|
||||
await worker.stop().catch(() => undefined)
|
||||
await queue.close().catch(() => undefined)
|
||||
await connection.quit().catch(() => undefined)
|
||||
await platform.usage.close().catch(() => undefined)
|
||||
await data.close()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The worker process of a dialect profile: module jobs on BullMQ, the module cron (the BullMQ job
|
||||
* schedulers of dialect/schedules.ts, installed when the worker starts) and the outbox relay. There is no
|
||||
* role to check (one login); the MongoDB engine refused a standalone server while it connected
|
||||
* (AT-D15).
|
||||
*/
|
||||
export async function serveDialect(env: AppEnv, observe: ServerObservability, heartbeatFile: string): Promise<void> {
|
||||
const runtime = await buildDialectWorkerRuntime(env, observe)
|
||||
await runtime.worker.start()
|
||||
runtime.relay.start()
|
||||
const beat = () => writeFile(heartbeatFile, String(Date.now())).catch(() => undefined)
|
||||
await beat()
|
||||
observe.log.info(`worker: serving ${runtime.worker.jobs.length} job(s) on ${runtime.profile}`)
|
||||
// The module cron this profile installed on the queue at start (dialect/schedules.ts), named so an operator can read it.
|
||||
const cron = dialectSchedules(runtime.registry, runtime.profile).map((schedule) => schedule.job)
|
||||
observe.log.info(`worker: module cron on ${runtime.profile}: ${cron.length > 0 ? cron.join(", ") : "none"}`)
|
||||
const timer = setInterval(() => void beat(), 5_000)
|
||||
timer.unref()
|
||||
// The live change log keeps ids for 24 hours (compliance: retention.md), where the engine has one.
|
||||
const engine = engineOf(env)
|
||||
const prune = () =>
|
||||
engine
|
||||
.pruneLive?.(process.env, 24)
|
||||
.catch((error: unknown) => observe.log.warn(`worker: the live log was not pruned: ${(error as Error).name}`))
|
||||
void prune()
|
||||
const pruning = setInterval(() => void prune(), 3_600_000)
|
||||
pruning.unref()
|
||||
let stopping = false
|
||||
const stop = (signal: string): void => {
|
||||
if (stopping) return
|
||||
stopping = true
|
||||
observe.log.info(`worker: ${signal}, finishing the jobs in flight`)
|
||||
void (async () => {
|
||||
clearInterval(timer)
|
||||
clearInterval(pruning)
|
||||
await runtime.close()
|
||||
await observe.shutdown()
|
||||
process.exit(0)
|
||||
})()
|
||||
}
|
||||
process.on("SIGTERM", () => stop("SIGTERM"))
|
||||
process.on("SIGINT", () => stop("SIGINT"))
|
||||
}
|
||||
|
||||
/** The migrate job of a dialect profile: the one MARIADB_URL or MONGODB_URL, no owner and no grants. */
|
||||
export function migrateDialect(env: AppEnv): Promise<{ applied: string[]; skipped: string[] }> {
|
||||
const engine = engineOf(env)
|
||||
return engine.migrate(process.env, profileRegistry(appModules, engine.profile))
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The module cron of a mariadb or mongodb worker (G3j): the same declarations as the Postgres worker
|
||||
// (apps/worker/src/schedules.ts), kept to the modules that are installed on this profile. A schedule of a
|
||||
// module the profile does not ship (the agent module is Postgres-only) is left out, not scheduled to fail
|
||||
// every tick: the registry's repositories say which modules a profile has.
|
||||
|
||||
import type { DialectProfile } from "@app/config"
|
||||
import type { Registry } from "@conusai-kit/core"
|
||||
import type { ModuleSchedule } from "@conusai-kit/jobs-pgboss"
|
||||
import { appSchedules } from "../schedules.js"
|
||||
|
||||
export function dialectSchedules(
|
||||
registry: Registry,
|
||||
profile: DialectProfile,
|
||||
env: Record<string, string | undefined> = process.env,
|
||||
): readonly ModuleSchedule[] {
|
||||
return appSchedules(env).filter((schedule) => registry.repositories[schedule.module]?.[profile] !== undefined)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// Development on the fake platform only. Before an app tenant's business procedures can run, a
|
||||
// project administrator binds the tenant to the deployment and approves the exact service scopes the
|
||||
// app's modules declare; in Cloud those are the administrator's explicit acts, and a lookup never
|
||||
// creates a binding. A development run does the same for its demo organizations, through the fake
|
||||
// platform's provisioning route, with the scopes of the modules this app selected: an app that
|
||||
// selects no business module binds nothing (docs/how-to/test-business-services-locally.md).
|
||||
|
||||
import type { AppEnv } from "@app/config"
|
||||
import type { Registry } from "@conusai-kit/core"
|
||||
|
||||
/** Every scope the selected modules' business procedures declare, by service. */
|
||||
export function declaredBusinessScopes(registry: Pick<Registry, "modules">): Record<string, string[]> {
|
||||
const declared = new Map<string, Set<string>>()
|
||||
for (const module of registry.modules) {
|
||||
for (const requirement of module.serviceRequirements ?? []) {
|
||||
if (requirement.transport !== "business") continue
|
||||
const scopes = declared.get(requirement.service) ?? new Set<string>()
|
||||
for (const operation of Object.values(requirement.operations)) for (const scope of operation) scopes.add(scope)
|
||||
declared.set(requirement.service, scopes)
|
||||
}
|
||||
}
|
||||
return Object.fromEntries([...declared].map(([service, scopes]) => [service, [...scopes].sort()]))
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds each app tenant (an organization's id) on the fake platform with the declared scopes, and
|
||||
* answers how many it bound. Refused anywhere but a development run on the fake platform.
|
||||
*/
|
||||
export async function bindOnFakePlatform(
|
||||
env: Pick<AppEnv, "platform" | "production" | "deployment">,
|
||||
registry: Pick<Registry, "modules">,
|
||||
tenants: readonly string[],
|
||||
): Promise<number> {
|
||||
if (env.platform !== "fake" || env.production) {
|
||||
throw new Error(
|
||||
"business bindings are an administrator's act: only a development run on the fake platform provisions them",
|
||||
)
|
||||
}
|
||||
const grants = declaredBusinessScopes(registry)
|
||||
if (Object.keys(grants).length === 0) return 0
|
||||
const route = new URL("/__e2e/app-bindings", env.deployment.CONUSAI_API_URL)
|
||||
for (const tenant of tenants) {
|
||||
const response = await fetch(route, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ app_tenant_id: tenant, grants }),
|
||||
})
|
||||
await response.body?.cancel()
|
||||
if (!response.ok) throw new Error(`the fake platform refused a demo organization's binding (${response.status})`)
|
||||
}
|
||||
return tenants.length
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// kit:guides -- the guides module's build step; `kit template:prune --without guides` removes this file.
|
||||
//
|
||||
// Writes the app's code guides and its anchor catalogue into the image as one declarative, public
|
||||
// file (ADR-045): `node apps/worker/dist/guides-bundle.js <out> [anchors]`, run by the Dockerfile
|
||||
// after `kit guides:anchors`. conusai.json names <out> under `guides.bundle`, and after a healthy
|
||||
// deploy the platform reads it out of the image and publishes it, with a credential no container
|
||||
// ever holds. An invalid code guide or a broken catalogue fails the build.
|
||||
|
||||
import { existsSync } from "node:fs"
|
||||
import { resolve } from "node:path"
|
||||
import { appModules } from "@app/modules"
|
||||
import { buildRegistry } from "@conusai-kit/core"
|
||||
import { writeGuidesBundle } from "@conusai-kit/guides/deploy"
|
||||
|
||||
const [out, anchors] = process.argv.slice(2)
|
||||
if (!out) {
|
||||
console.error("guides-bundle: usage: node apps/worker/dist/guides-bundle.js <absolute out path> [anchors file]")
|
||||
process.exit(64)
|
||||
}
|
||||
const anchorsFile = anchors && existsSync(anchors) ? resolve(anchors) : undefined
|
||||
const bundle = writeGuidesBundle(buildRegistry([...appModules]), anchorsFile, out)
|
||||
console.info(
|
||||
`guides-bundle: wrote ${bundle.guides.length} code guide(s) and ${bundle.anchors.length} anchor(s) to ${out}`,
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The worker's healthcheck (compose.yaml). It reads the heartbeat the process writes rather than
|
||||
// opening a database connection: a healthcheck that connects to Postgres every fifteen seconds
|
||||
// reports the database's health, not the worker's, and holds a connection the app could use.
|
||||
|
||||
import { readFile } from "node:fs/promises"
|
||||
|
||||
const STALE_AFTER_MS = 45_000
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const file = process.env.WORKER_HEARTBEAT_FILE ?? "/tmp/worker-heartbeat"
|
||||
const written = Number.parseInt(await readFile(file, "utf8"), 10)
|
||||
if (!Number.isFinite(written)) throw new Error(`${file} holds no timestamp`)
|
||||
const age = Date.now() - written
|
||||
if (age > STALE_AFTER_MS) throw new Error(`the last heartbeat is ${Math.round(age / 1000)}s old`)
|
||||
}
|
||||
|
||||
main().then(
|
||||
() => process.exit(0),
|
||||
(error: unknown) => {
|
||||
console.error(`worker healthcheck: ${error instanceof Error ? error.message : String(error)}`)
|
||||
process.exit(1)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// What the worker tells observability about its work: `@conusai-kit/host/worker`, on every profile.
|
||||
|
||||
export { logWorkerEvent, traceJobs } from "@conusai-kit/host/worker"
|
||||
@@ -0,0 +1,59 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The worker process: module jobs, module cron and the outbox relay (spec §8), which is
|
||||
// `createPostgresWorker` given this app's modules, ports and schedules. The mariadb and mongodb
|
||||
// profiles serve their own (dialect/runtime.ts).
|
||||
|
||||
import {
|
||||
businessServiceComposition,
|
||||
currentObservability,
|
||||
loadAppEnv,
|
||||
startObservability, // kit:profile-dialect
|
||||
} from "@app/config"
|
||||
import { appModules } from "@app/modules"
|
||||
import { createPostgresWorker } from "@conusai-kit/host/worker"
|
||||
import { serveDialect } from "./dialect/runtime.js" // kit:profile-dialect
|
||||
import { bindModulePorts, nativeHelpSelectors } from "./ports.js"
|
||||
import { appSchedules } from "./schedules.js"
|
||||
|
||||
/** The file the healthcheck reads. The relay touches it on every pass, so a stuck relay is visible. */
|
||||
const heartbeatFile = (): string => process.env.WORKER_HEARTBEAT_FILE ?? "/tmp/worker-heartbeat" // kit:profile-dialect
|
||||
|
||||
async function main(): Promise<void> {
|
||||
// Before anything else is built: a missing platform variable or an owner's URL stops the worker
|
||||
// here, with a typed error that names it, and nothing has connected to anything yet.
|
||||
// kit:profile-dialect:begin
|
||||
const env = loadAppEnv("worker")
|
||||
if (env.mariadb || env.mongodb) return serveDialect(env, startObservability(env), heartbeatFile())
|
||||
// kit:profile-dialect:end
|
||||
const workerEnv = loadAppEnv("worker")
|
||||
const worker = createPostgresWorker({
|
||||
modules: appModules,
|
||||
bindPorts: (root, { platform, registry }) =>
|
||||
bindModulePorts(
|
||||
root,
|
||||
platform,
|
||||
registry,
|
||||
businessServiceComposition(workerEnv, appModules, "postgres", { model: platform.model, ...nativeHelpSelectors })
|
||||
.serviceReadiness,
|
||||
),
|
||||
schedules: appSchedules(),
|
||||
})
|
||||
await worker.start()
|
||||
|
||||
let stopping = false
|
||||
const stop = (signal: string): void => {
|
||||
if (stopping) return
|
||||
stopping = true
|
||||
currentObservability()?.log.info(`worker: ${signal}, finishing the jobs in flight`)
|
||||
void worker.close().finally(() => process.exit(0))
|
||||
}
|
||||
process.on("SIGTERM", () => stop("SIGTERM"))
|
||||
process.on("SIGINT", () => stop("SIGINT"))
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
// Nothing may have started when this runs (a bad environment), so it is written plainly: the
|
||||
// message names the variable, and a typed error carries no secret by construction.
|
||||
console.error(`worker: ${error instanceof Error ? error.message : String(error)}`)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The migrate job (conusai.json: `"migrate": { "kind": "job", "postgres": { "migrate": true } }`).
|
||||
//
|
||||
// It is the only process that holds POSTGRES_MIGRATE_URL, the owner's URL, and the only one that
|
||||
// writes the schema (decision AT-D20). Three things happen here, in this order:
|
||||
//
|
||||
// 1 to 3. `runAppMigrations` (@conusai-kit/host/migrate): the app's own forward-only migrations from
|
||||
// migrations/postgres, every module's declared folder, and the job adapter's schema with the
|
||||
// grants that let the runtime role use it and the global reader read across tenants
|
||||
// kit:guides:begin
|
||||
// 4. with the guides module, the DEPRECATED compose hand-off of the code guides and the anchor
|
||||
// catalogue. This app declares guides.bundle in conusai.json, so the platform publishes the
|
||||
// bundle the image build wrote (apps/worker/src/guides-bundle.ts, ADR-045) and hands this job
|
||||
// no credential: the step logs that it skipped. It stays only until the compose hand-off is
|
||||
// removed, for an app that still has no guides.bundle.
|
||||
// kit:guides:end
|
||||
// kit:convex-better-auth:begin
|
||||
// 5. with AUTH_PROVIDER=convex-better-auth, the Better Auth functions (better-auth/), deployed to
|
||||
// the app's Better Auth Convex deployment with its SITE_URL and secret (better-auth/deploy.mjs)
|
||||
// kit:convex-better-auth:end
|
||||
//
|
||||
// It exits non-zero when anything fails, which is what web and worker wait on:
|
||||
// `depends_on: { migrate: { condition: service_completed_successfully } }` in compose.yaml
|
||||
// (decision AT-D14 -- `kind: job` alone does not order a deployment).
|
||||
|
||||
import { spawnSync } from "node:child_process" // kit:convex-better-auth
|
||||
import { existsSync } from "node:fs" // kit:guides
|
||||
import { fileURLToPath } from "node:url" // kit:guides
|
||||
import { appRoot, loadAppEnv } from "@app/config"
|
||||
import { appModules } from "@app/modules"
|
||||
import { moduleSelected } from "@app/modules/inclusion"
|
||||
import { buildRegistry } from "@conusai-kit/core" // kit:guides
|
||||
import { describeOutcome, publishGuidesAtDeploy } from "@conusai-kit/guides/deploy" // kit:guides
|
||||
import { runAppMigrations } from "@conusai-kit/host/migrate"
|
||||
import { migrateDialect } from "./dialect/runtime.js" // kit:profile-dialect
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const env = loadAppEnv("migrate")
|
||||
// kit:profile-dialect:begin
|
||||
if (env.mariadb || env.mongodb) {
|
||||
// mariadb and mongodb: the kernel's dialect files, the job adapter's table, then every module's
|
||||
// declared folder, on the one login (no owner, no grants: there is no RLS to make bind).
|
||||
const outcome = await migrateDialect(env)
|
||||
console.info(`migrate: applied ${outcome.applied.join(", ") || "nothing"}; skipped ${outcome.skipped.length}`)
|
||||
return
|
||||
}
|
||||
// kit:profile-dialect:end
|
||||
const postgres = env.postgres
|
||||
if (!postgres) throw new Error("the selected profile has no Postgres migrations")
|
||||
const owner = postgres.owner
|
||||
if (!owner) throw new Error("the migrate job has no owner URL, which loadAppEnv refuses before this line")
|
||||
|
||||
await runAppMigrations({ ownerUrl: owner, modules: appModules, root: appRoot() })
|
||||
// kit:guides:begin -- after the schema, so a refused publication never leaves a half-migrated
|
||||
// database; a refusal is logged and the deployment goes on (the console shows the guide's state).
|
||||
if (moduleSelected("guides")) {
|
||||
const anchors = new URL("../../../guides-anchors.json", import.meta.url)
|
||||
const guides = await publishGuidesAtDeploy({
|
||||
registry: buildRegistry([...appModules]),
|
||||
env: {
|
||||
CONUSAI_API_URL: process.env.CONUSAI_API_URL,
|
||||
CONUSAI_GUIDES_PUBLISH_TOKEN: process.env.CONUSAI_GUIDES_PUBLISH_TOKEN,
|
||||
CONUSAI_GUIDES_DEPLOYMENT: process.env.CONUSAI_GUIDES_DEPLOYMENT,
|
||||
},
|
||||
...(existsSync(anchors) ? { anchorsFile: fileURLToPath(anchors) } : {}),
|
||||
})
|
||||
console.info(`migrate: ${describeOutcome(guides)}`)
|
||||
}
|
||||
// kit:guides:end
|
||||
// kit:convex-better-auth:begin -- a failed deploy fails the job, so web and worker never start
|
||||
// against a Better Auth deployment that cannot answer their session checks.
|
||||
if (process.env.AUTH_PROVIDER?.trim() === "convex-better-auth") {
|
||||
const script = new URL("../../../better-auth/deploy.mjs", import.meta.url).pathname
|
||||
const deployed = spawnSync(process.execPath, [script], { env: process.env, stdio: "inherit" })
|
||||
if (deployed.error || deployed.status !== 0) throw new Error("the Better Auth functions were not deployed")
|
||||
}
|
||||
// kit:convex-better-auth:end
|
||||
}
|
||||
|
||||
main().then(
|
||||
() => process.exit(0),
|
||||
(error: unknown) => {
|
||||
console.error(`migrate: ${error instanceof Error ? error.message : String(error)}`)
|
||||
process.exit(1)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The app's forward-only migrations for the Postgres profile (spec §13) are `@conusai-kit/host/migrate`.
|
||||
|
||||
export {
|
||||
CROSS_TENANT_READS,
|
||||
jobSchemaGrants,
|
||||
kernelSchema,
|
||||
moduleMigrations,
|
||||
readerGrants,
|
||||
splitStatements,
|
||||
sqlMigrations,
|
||||
} from "@conusai-kit/host/migrate"
|
||||
@@ -0,0 +1,60 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// Current app intent + native service facts for actual claimed worker effects.
|
||||
import { admitServices, type Container, type Registry, type ServiceReadiness, serviceRefusal } from "@conusai-kit/core"
|
||||
import { DATA_SOURCE } from "@conusai-kit/data"
|
||||
import { JOB_ON_BEHALF_OF, JOB_SCOPE } from "@conusai-kit/jobs-pgboss"
|
||||
import { NATIVE_JOB_ADMISSION } from "@conusai-kit/platform"
|
||||
import { authorizeJobIntent } from "@conusai-kit/tenancy"
|
||||
|
||||
/**
|
||||
* Each module job that leaves the process, and the permissions the person it acts for must still hold
|
||||
* when it runs (test/native-admission.test.ts: every such job has one).
|
||||
*/
|
||||
export const JOB_INTENT_PERMISSIONS: Readonly<Record<string, readonly string[]>> = {
|
||||
"documents.ocr": ["files.file.write", "documents.document.read"],
|
||||
"knowledge.index": ["knowledge.knowledge.manage"],
|
||||
// The automatic PDF intake acts for the administrator who opted the project in (core-11).
|
||||
"knowledge.autoindex": ["knowledge.knowledge.manage"],
|
||||
}
|
||||
|
||||
export function bindNativeJobAdmission(root: Container, registry: Registry, readiness?: ServiceReadiness): void {
|
||||
root.register(
|
||||
NATIVE_JOB_ADMISSION,
|
||||
(container) => ({
|
||||
async authorize(scope, job, subjectUserId, signal) {
|
||||
signal.throwIfAborted()
|
||||
const actual = container.resolve(JOB_SCOPE)
|
||||
if (
|
||||
actual.level !== "project" ||
|
||||
scope.level !== "project" ||
|
||||
actual.actor.system !== true ||
|
||||
actual.actor.userId !== `system:${job}` ||
|
||||
actual.orgId !== scope.orgId ||
|
||||
actual.workspaceId !== scope.workspaceId ||
|
||||
actual.projectId !== scope.projectId
|
||||
)
|
||||
throw serviceRefusal("permission_missing")
|
||||
const owner = registry.jobs[job]?.module
|
||||
const permissions = JOB_INTENT_PERMISSIONS[job]
|
||||
const requirements = registry.modules
|
||||
.find((module) => module.id === owner)
|
||||
?.serviceRequirements?.filter(
|
||||
(requirement) => requirement.transport === "platform" && Object.hasOwn(requirement.operations, job),
|
||||
)
|
||||
if (!owner || !permissions?.length || !requirements?.length) throw serviceRefusal("observation_missing")
|
||||
const original = container.resolve(JOB_ON_BEHALF_OF)
|
||||
if (original !== null && original !== subjectUserId) throw serviceRefusal("permission_missing")
|
||||
for (const permission of permissions)
|
||||
await authorizeJobIntent(container.resolve(DATA_SOURCE), registry, scope, subjectUserId, permission)
|
||||
signal.throwIfAborted()
|
||||
await admitServices(
|
||||
requirements,
|
||||
{ container, module: owner, procedure: job, kind: "effect", scope },
|
||||
readiness,
|
||||
)
|
||||
signal.throwIfAborted()
|
||||
},
|
||||
}),
|
||||
{ lifetime: "scoped" },
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The ports the kit's modules and this app's modules resolve by token, bound for the worker.
|
||||
//
|
||||
// A module resolves a port by token and never imports an adapter (spec §5.6), so the adapters are
|
||||
// chosen here, in the composition root, from the deployment contract. The platform's services
|
||||
// themselves are built by `platformClients` (config/src/platform.ts), the same function the web
|
||||
// shell calls, so the two processes cannot talk to different services. Three kinds of binding:
|
||||
//
|
||||
// - the platform's own services: Blob for the files module's object store, the audit log and the
|
||||
// usage meter for the subscribers, the OCR service and the AI gateway for the documents job;
|
||||
// - the registry, which the files and proposals modules read their extension points from;
|
||||
// - the scope of a job or an outbox delivery, which the worker binds per run as `JOB_SCOPE`
|
||||
// (kit-jobs-pgboss) and the modules that cannot import that adapter resolve through their own
|
||||
// token (decision AT-F6). An alias is a factory that reads the per-run binding, so it is
|
||||
// registered once, on the process's container.
|
||||
|
||||
import {
|
||||
currentObservability,
|
||||
excludedPageClassNamesOf, // kit:documents
|
||||
type PlatformClients,
|
||||
} from "@app/config"
|
||||
import { moduleAdopted, moduleSelected } from "@app/modules"
|
||||
import { ACCOUNT_LOG, ACCOUNT_PRIVACY, ACCOUNT_REGISTRY } from "@app/modules/account"
|
||||
import * as agent from "@app/modules/agent" // kit:agent
|
||||
import * as documents from "@app/modules/documents" // kit:documents
|
||||
import * as knowledge from "@app/modules/knowledge" // kit:knowledge
|
||||
import type { Container, Registry, ServiceReadiness } from "@conusai-kit/core"
|
||||
import { systemActor } from "@conusai-kit/core/worker"
|
||||
import { OBJECT_STORE } from "@conusai-kit/data"
|
||||
import {
|
||||
FILES_AUDIT,
|
||||
FILES_REGISTRY,
|
||||
FILES_SCOPE,
|
||||
FILES_USAGE,
|
||||
type FilesAuditSink,
|
||||
filesDeps, // kit:knowledge
|
||||
platformObjectStore,
|
||||
} from "@conusai-kit/files"
|
||||
import { GUIDES_PLATFORM, GUIDES_REGISTRY, guidesPlatform } from "@conusai-kit/guides" // kit:guides
|
||||
import { JOB_REPOS, JOB_SCOPE } from "@conusai-kit/jobs-pgboss"
|
||||
import { NOTIFICATIONS_REPOS, NOTIFICATIONS_RETENTION_ACTOR } from "@conusai-kit/notifications"
|
||||
import {
|
||||
createKnowledge, // kit:knowledge
|
||||
createPrivacy,
|
||||
} from "@conusai-kit/platform"
|
||||
import { PROPOSALS_AUDIT, PROPOSALS_REGISTRY, PROPOSALS_SCOPE } from "@conusai-kit/proposals"
|
||||
import { SUPPORT_PLATFORM, supportPlatform } from "@conusai-kit/support" // kit:support
|
||||
import { bindBetterAuthAccountEnd } from "./better-auth.js" // kit:convex-better-auth
|
||||
import { bindNativeJobAdmission } from "./native-admission.js"
|
||||
|
||||
export function bindModulePorts(
|
||||
container: Container,
|
||||
platform: PlatformClients,
|
||||
registry: Registry,
|
||||
nativeReadiness?: ServiceReadiness,
|
||||
): void {
|
||||
bindNativeJobAdmission(container, registry, nativeReadiness)
|
||||
container.value(OBJECT_STORE, platformObjectStore({ blob: platform.objectStore }))
|
||||
container.value(FILES_REGISTRY, registry)
|
||||
container.value(PROPOSALS_REGISTRY, registry)
|
||||
// The account module's sweep resumes a deletion a module's erasure held up: it reads the installed
|
||||
// modules' shares from the registry and reaches their services through the same ports the web binds.
|
||||
container.value(ACCOUNT_REGISTRY, registry)
|
||||
// Only worker/relay containers receive these capabilities; requests cannot send inbox messages.
|
||||
container.register(NOTIFICATIONS_REPOS, (c) => c.resolve(JOB_REPOS), { lifetime: "scoped" })
|
||||
container.value(NOTIFICATIONS_RETENTION_ACTOR, systemActor("notifications.retention.sweep"))
|
||||
// Why a share failed while a deletion or a revocation was retried: the operator's, redacted like every other line.
|
||||
container.value(ACCOUNT_LOG, { warn: (message) => currentObservability()?.log.warn(message) })
|
||||
const privacy = createPrivacy(platform.deployment)
|
||||
container.value(ACCOUNT_PRIVACY, { ...privacy, ticketKey: platform.deployment.CONUSAI_API_TOKEN })
|
||||
// The sweep resumes a deletion at the step that ends the person's Better Auth account.
|
||||
bindBetterAuthAccountEnd(container, process.env) // kit:convex-better-auth
|
||||
if (moduleAdopted("guides")) {
|
||||
// kit:guides:begin -- the guides module's erasure, for a resumed account deletion.
|
||||
container.value(GUIDES_REGISTRY, registry)
|
||||
container.register(GUIDES_PLATFORM, () =>
|
||||
guidesPlatform({
|
||||
apiUrl: platform.deployment.CONUSAI_API_URL,
|
||||
token: platform.deployment.CONUSAI_API_TOKEN,
|
||||
fake: platform.mode === "fake",
|
||||
}),
|
||||
)
|
||||
// kit:guides:end
|
||||
}
|
||||
if (moduleAdopted("support")) {
|
||||
// kit:support:begin -- the support module's erasure, for a resumed account deletion.
|
||||
container.register(SUPPORT_PLATFORM, () =>
|
||||
supportPlatform({
|
||||
apiUrl: platform.deployment.CONUSAI_API_URL,
|
||||
token: platform.deployment.CONUSAI_API_TOKEN,
|
||||
channel: process.env.CONUSAI_SUPPORT_CHANNEL,
|
||||
fake: platform.mode === "fake",
|
||||
}),
|
||||
)
|
||||
// kit:support:end
|
||||
}
|
||||
|
||||
// The modules' sinks take the action as a plain string; the registry the audit client was built
|
||||
// from (config's `auditEventsOf`) is what makes it a known one, and the client refuses the rest.
|
||||
const sink: FilesAuditSink = {
|
||||
log: (action, fields) => platform.audit.log(action, fields as Parameters<typeof platform.audit.log>[1]),
|
||||
}
|
||||
container.value(FILES_AUDIT, sink)
|
||||
container.value(PROPOSALS_AUDIT, sink)
|
||||
|
||||
container.value(FILES_USAGE, {
|
||||
track: (event) => platform.usage.track(event),
|
||||
flush: async () => {
|
||||
await platform.usage.flush()
|
||||
},
|
||||
})
|
||||
|
||||
if (moduleSelected("documents")) {
|
||||
// kit:documents:begin
|
||||
// The platform's asynchronous OCR job API: a job submits and polls, a request never does.
|
||||
container.value(documents.DOCUMENT_OCR, documents.sdkDocumentOcr(platform.documents))
|
||||
// Document Items remain bound for the selected module (ADR-069).
|
||||
container.value(documents.DOCUMENT_ITEMS, platform.documents.items)
|
||||
|
||||
// The AI gateway the documents job extracts lines through, and the model it asks for (AI_MODEL).
|
||||
container.value(documents.MODEL_GATEWAY, platform.models)
|
||||
container.value(documents.DOCUMENT_MODEL, platform.model)
|
||||
// The page classes the extraction never asks the model about; unset, the module's default (drawings and empty pages).
|
||||
const excluded = excludedPageClassNamesOf()
|
||||
if (excluded) container.value(documents.DOCUMENT_EXCLUDED_PAGE_CLASSES, documents.pageClassesOf(excluded))
|
||||
// kit:documents:end
|
||||
}
|
||||
|
||||
// The per-run scope, under each module's own name for it.
|
||||
for (const alias of [
|
||||
FILES_SCOPE,
|
||||
PROPOSALS_SCOPE,
|
||||
documents.DOCUMENTS_SCOPE, // kit:documents
|
||||
knowledge.KNOWLEDGE_SCOPE, // kit:knowledge
|
||||
agent.AGENT_SCOPE, // kit:agent
|
||||
agent.ASSISTANT_SCOPE, // kit:agent
|
||||
]) {
|
||||
container.register(alias, (c) => c.resolve(JOB_SCOPE), { lifetime: "scoped" })
|
||||
}
|
||||
|
||||
// The knowledge module's search backend and the agent runtime's tools, bound for the worker.
|
||||
if (moduleAdopted("knowledge")) bindKnowledgePorts(container, platform) // kit:knowledge
|
||||
if (moduleSelected("agent")) bindAgentPorts(container, platform) // kit:agent
|
||||
}
|
||||
|
||||
// kit:knowledge:begin
|
||||
/**
|
||||
* The knowledge module's ports: the platform's knowledge service (ADR-061), and for the automatic PDF intake
|
||||
* (core-11) the platform's document reader, the service's inline index and the enablement checks it repeats
|
||||
* before each read. Indexing and removal jobs run in this process only; a request writes the outbox event
|
||||
* and answers. Lazy, so a boot constructs none of them.
|
||||
*/
|
||||
function bindKnowledgePorts(container: Container, platform: PlatformClients): void {
|
||||
container.register(knowledge.KNOWLEDGE_SEARCH, (c) =>
|
||||
knowledge.platformKnowledgeSearch(createKnowledge(platform.deployment), filesDeps(c), {
|
||||
environment: platform.deployment.CONUSAI_ENVIRONMENT,
|
||||
}),
|
||||
)
|
||||
container.register(knowledge.KNOWLEDGE_READER, () => knowledge.sdkKnowledgeReader(platform.documents))
|
||||
container.register(knowledge.KNOWLEDGE_ENABLEMENT, () =>
|
||||
knowledge.sdkDocumentAiPlatform({
|
||||
apiUrl: platform.deployment.CONUSAI_API_URL,
|
||||
token: platform.deployment.CONUSAI_API_TOKEN,
|
||||
}),
|
||||
)
|
||||
container.register(knowledge.KNOWLEDGE_PASSAGES, (c) =>
|
||||
knowledge.platformPassageIndex(createKnowledge(platform.deployment), filesDeps(c), {
|
||||
environment: platform.deployment.CONUSAI_ENVIRONMENT,
|
||||
}),
|
||||
)
|
||||
}
|
||||
// kit:knowledge:end
|
||||
|
||||
// kit:agent:begin
|
||||
/**
|
||||
* The agent runtime's tools (the organization's registry, with the app's scoped credentials) and
|
||||
* its model (the AI gateway's, so metering and the spend cap stay in the gateway).
|
||||
*/
|
||||
function bindAgentPorts(container: Container, platform: PlatformClients): void {
|
||||
container.value(
|
||||
agent.AGENT_MODEL,
|
||||
agent.gatewayAgentModel({
|
||||
apiUrl: platform.deployment.CONUSAI_API_URL,
|
||||
token: platform.deployment.CONUSAI_API_TOKEN,
|
||||
model: platform.model,
|
||||
}),
|
||||
)
|
||||
container.value(
|
||||
agent.AGENT_TOOLS,
|
||||
agent.registryAgentTools({
|
||||
apiUrl: platform.deployment.CONUSAI_API_URL,
|
||||
token: platform.deployment.CONUSAI_API_TOKEN,
|
||||
orgId: platform.deployment.CONUSAI_ORG_ID ?? "0",
|
||||
// Resolved per call: jobs of different tenants share this one process binding, and the
|
||||
// registry's tool-actions API is addressed by slugs, not ids.
|
||||
scope: async () => {
|
||||
const scope = container.resolve(agent.AGENT_SCOPE)
|
||||
if (scope.level !== "project") throw new Error("agent tools resolve in a project scope")
|
||||
return agent.actionScopeOf(container.resolve(agent.AGENT_DATA), scope)
|
||||
},
|
||||
}),
|
||||
)
|
||||
}
|
||||
// kit:agent:end
|
||||
|
||||
// Read the server-selected runtime ports at authorized dispatch, never browser input.
|
||||
export const nativeHelpSelectors = {
|
||||
// kit:support:begin
|
||||
supportChannel: (container: Container): string | undefined => {
|
||||
if (!container.has(SUPPORT_PLATFORM)) return undefined
|
||||
const port = container.resolve(SUPPORT_PLATFORM)
|
||||
return port.mode === "platform" ? port.channel : undefined
|
||||
},
|
||||
// kit:support:end
|
||||
// kit:guides:begin
|
||||
guidesIdentityNamespace: (container: Container): string | undefined => {
|
||||
if (!container.has(GUIDES_PLATFORM)) return undefined
|
||||
const port = container.resolve(GUIDES_PLATFORM)
|
||||
return port.mode === "platform" ? port.identityNamespace : undefined
|
||||
},
|
||||
// kit:guides:end
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The worker's composition root for what runs beside `createPostgresWorker` (spec §2): this app's modules
|
||||
// and ports on `@conusai-kit/host/worker`. The seed job builds on it.
|
||||
|
||||
import { type AppEnv, businessServiceComposition, type ServerObservability } from "@app/config"
|
||||
import { appModules } from "@app/modules"
|
||||
import { buildPostgresWorkerRuntime, type WorkerRuntime } from "@conusai-kit/host/worker"
|
||||
import { bindModulePorts, nativeHelpSelectors } from "./ports.js"
|
||||
|
||||
export type { WorkerRuntime }
|
||||
|
||||
export function buildWorkerRuntime(env: AppEnv, observe: ServerObservability): WorkerRuntime {
|
||||
const postgres = env.postgres
|
||||
if (!postgres) throw new Error("The selected profile has no Postgres worker")
|
||||
const runtime = buildPostgresWorkerRuntime({ modules: appModules, env: { ...env, postgres }, observe })
|
||||
bindModulePorts(
|
||||
runtime.container,
|
||||
runtime.platform,
|
||||
runtime.registry,
|
||||
businessServiceComposition(env, appModules, "postgres", { model: runtime.platform.model, ...nativeHelpSelectors })
|
||||
.serviceReadiness,
|
||||
)
|
||||
return runtime
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The module cron of this app: what the worker runs on a clock, not because anyone asked. A module declares
|
||||
// it in its manifest (`schedules`), so a new module edits nothing here, and `installSchedules` refuses at boot
|
||||
// a schedule of a job no module declares.
|
||||
//
|
||||
// The same list serves both adapters: pg-boss on Postgres, and the BullMQ job schedulers of the mariadb and
|
||||
// mongodb profiles, which take only the modules installed on that profile (dialect/schedules.ts).
|
||||
|
||||
import { appModules } from "@app/modules"
|
||||
import { type ModuleSchedule, moduleSchedules } from "@conusai-kit/jobs-pgboss"
|
||||
|
||||
export function appSchedules(env: Record<string, string | undefined> = process.env): readonly ModuleSchedule[] {
|
||||
return moduleSchedules(appModules, env)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// `pnpm seed`: the demo data (seed.ts) written into the database a development run uses. It runs as
|
||||
// the runtime role, like the web shell, through the same use cases, so the schema must be migrated
|
||||
// first (`pnpm migrate`; `pnpm dev` does both). It refuses a production run: demo people with known
|
||||
// addresses in a real deployment's database is a mistake no flag should make easy. On the fake
|
||||
// platform it then binds the demo organizations to the business services the app's modules use.
|
||||
|
||||
import { loadAppEnv, PlatformEnvError, quietObservability } from "@app/config"
|
||||
import { organizations } from "@conusai-kit/tenancy"
|
||||
import { bindOnFakePlatform } from "./fake-business-bindings.js"
|
||||
import { buildWorkerRuntime } from "./runtime.js"
|
||||
import { DEMO_PEOPLE, seedDemo } from "./seed.js"
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const env = loadAppEnv("worker")
|
||||
if (env.production) {
|
||||
throw new PlatformEnvError(
|
||||
"NODE_ENV",
|
||||
"the demo data is for development: it is refused in a production run (NODE_ENV is neither development nor test)",
|
||||
)
|
||||
}
|
||||
const runtime = buildWorkerRuntime(env, quietObservability(env))
|
||||
try {
|
||||
const summary = await seedDemo({ data: runtime.data, registry: runtime.registry, container: runtime.container })
|
||||
const base = env.deployment.CONUSAI_PUBLIC_URL ?? "http://127.0.0.1:3000"
|
||||
console.info(
|
||||
summary.created.length === 0
|
||||
? "seed: the demo data was already there"
|
||||
: `seed: created ${summary.created.join("; ")}`,
|
||||
)
|
||||
for (const person of DEMO_PEOPLE) {
|
||||
console.info(`seed: ${person.name}: ${base}/auth/dev-login?email=${person.email}`)
|
||||
}
|
||||
if (env.platform === "fake") {
|
||||
const tenants: string[] = []
|
||||
for (const org of summary.organizations) {
|
||||
const page = await runtime.data
|
||||
.global(organizations)
|
||||
.list({ where: [{ field: "slug", op: "eq", value: org.slug }], limit: 1 })
|
||||
if (page.items[0]) tenants.push(page.items[0].id)
|
||||
}
|
||||
const bound = await bindOnFakePlatform(env, runtime.registry, tenants)
|
||||
if (bound > 0) console.info(`seed: bound ${bound} demo organizations to the fake platform's business services`)
|
||||
}
|
||||
} finally {
|
||||
await runtime.close()
|
||||
}
|
||||
}
|
||||
|
||||
main().then(
|
||||
() => process.exit(0),
|
||||
(error: unknown) => {
|
||||
console.error(`seed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
process.exit(1)
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
||||
// The demo data of a development run: two organizations, one of them with a tenant brand, three
|
||||
// people who belong to them in different roles, and a project with a few notes -- enough to see the
|
||||
// organization switcher, the roles, a branded organization and a module's pages the first time the
|
||||
// app starts.
|
||||
//
|
||||
// It is written with the kit and nothing else. People come into being the way they do at sign-in
|
||||
// (`upsertUser`, as `@conusai-kit/auth` calls it); everything after that is a procedure called as
|
||||
// the person who may call it, through the kernel's own caller, so the permission checks, the
|
||||
// last-owner rule, the invitation single use and the tenancy walls all apply to the seed exactly as
|
||||
// they apply to a request. There is no SQL here and no row written past a use case.
|
||||
//
|
||||
// Running it twice changes nothing: every step first asks whether it is already there.
|
||||
|
||||
import { DEV_LOGIN_ISSUER, upsertUser } from "@conusai-kit/auth"
|
||||
import { type Actor, type Container, createCaller, type Registry, type TransportCaller } from "@conusai-kit/core"
|
||||
import { DATA_SOURCE, type DataSource } from "@conusai-kit/data"
|
||||
import { createTenancyHooks, TENANCY } from "@conusai-kit/tenancy"
|
||||
|
||||
/** The people of the demo. They sign in at `/auth/dev-login?email=<email>` in a development run. */
|
||||
export const DEMO_PEOPLE = [
|
||||
{ email: "ada@example.test", name: "Ada Lovelace" },
|
||||
{ email: "grace@example.test", name: "Grace Hopper" },
|
||||
{ email: "linus@example.test", name: "Linus Torvalds" },
|
||||
] as const
|
||||
|
||||
export const DEMO_ORGANIZATIONS = {
|
||||
/** The default look: the app's own brand. */
|
||||
acme: { slug: "acme-review", name: "Acme Review" },
|
||||
/** A tenant brand: its own name and colours on every page. */
|
||||
northwind: {
|
||||
slug: "northwind",
|
||||
name: "Northwind Traders",
|
||||
brand: { displayName: "Northwind", primary: "#1a1a8c", accent: "#f4c430" },
|
||||
},
|
||||
} as const
|
||||
|
||||
const WORKSPACE = "general"
|
||||
const PROJECT = { slug: "demo", name: "Demo project" } // kit:documents
|
||||
|
||||
export interface SeedDeps {
|
||||
readonly data: DataSource
|
||||
readonly registry: Registry
|
||||
/** The process's container: the ports the modules' procedures resolve (Blob, the queue, ...). */
|
||||
readonly container: Container
|
||||
}
|
||||
|
||||
export interface SeedSummary {
|
||||
/** What this run created; empty when everything was already there. */
|
||||
readonly created: string[]
|
||||
/** The addresses to sign in at, by person. */
|
||||
readonly people: readonly { email: string; name: string }[]
|
||||
readonly organizations: readonly { slug: string; name: string }[]
|
||||
}
|
||||
|
||||
type Role = "viewer" | "member" | "admin" | "owner"
|
||||
|
||||
export async function seedDemo(deps: SeedDeps): Promise<SeedSummary> {
|
||||
const { data, registry } = deps
|
||||
const created: string[] = []
|
||||
|
||||
const actors = new Map<string, Actor>()
|
||||
for (const person of DEMO_PEOPLE) {
|
||||
const user = await upsertUser(data, {
|
||||
issuer: DEV_LOGIN_ISSUER,
|
||||
subject: person.email,
|
||||
email: person.email,
|
||||
displayName: person.name,
|
||||
locale: "en",
|
||||
})
|
||||
actors.set(person.email, { userId: user.id })
|
||||
}
|
||||
|
||||
/** A caller for one person, with its own tenancy hooks: the memo of roles must not outlive a call. */
|
||||
const as = (email: string): TransportCaller => {
|
||||
const actor = actors.get(email)
|
||||
if (!actor) throw new Error(`the demo has no person ${email}`)
|
||||
const tenancy = createTenancyHooks({ data, registry })
|
||||
const container = deps.container.child()
|
||||
container.value(DATA_SOURCE, data)
|
||||
container.value(TENANCY, tenancy)
|
||||
return createCaller({ registry, container, tenancy }, { actor })
|
||||
}
|
||||
|
||||
const rolesOf = async (email: string): Promise<Map<string, Role>> => {
|
||||
const me = (await as(email).callByName("tenancy.me.get", undefined)) as {
|
||||
organizations: { slug: string; role: Role }[]
|
||||
}
|
||||
return new Map(me.organizations.map((entry) => [entry.slug, entry.role]))
|
||||
}
|
||||
|
||||
/** Creates an organization for its owner unless they already have it. */
|
||||
const organization = async (owner: string, org: { slug: string; name: string }): Promise<void> => {
|
||||
if ((await rolesOf(owner)).has(org.slug)) return
|
||||
await as(owner).callByName("tenancy.org.create", { slug: org.slug, name: org.name })
|
||||
created.push(`organization ${org.slug}`)
|
||||
}
|
||||
|
||||
/** Brings a person into an organization the way anybody joins one: an invitation, accepted by them. */
|
||||
const member = async (owner: string, org: string, email: string, role: Role): Promise<void> => {
|
||||
if ((await rolesOf(email)).has(org)) return
|
||||
const invitation = (await as(owner).callByName("tenancy.member.invite", { org, email, role })) as { token: string }
|
||||
await as(email).callByName("tenancy.invitation.accept", { org, token: invitation.token })
|
||||
created.push(`${email} as ${role} in ${org}`)
|
||||
}
|
||||
|
||||
const project = async (owner: string, org: string, moduleKey: string, slug: string, name: string): Promise<void> => {
|
||||
const listed = (await as(owner).callByName("tenancy.project.list", { org, workspace: WORKSPACE })) as {
|
||||
projects: { slug: string }[]
|
||||
}
|
||||
if (listed.projects.some((entry) => entry.slug === slug)) return
|
||||
await as(owner).callByName("tenancy.project.create", { org, workspace: WORKSPACE, slug, name, moduleKey })
|
||||
created.push(`project ${org}/${WORKSPACE}/${slug}`)
|
||||
}
|
||||
|
||||
const { acme, northwind } = DEMO_ORGANIZATIONS
|
||||
const ada = "ada@example.test"
|
||||
const grace = "grace@example.test"
|
||||
const linus = "linus@example.test"
|
||||
|
||||
// Acme Review: Ada owns it, Grace works in it, Linus only reads.
|
||||
await organization(ada, acme)
|
||||
await member(ada, acme.slug, grace, "member")
|
||||
await member(ada, acme.slug, linus, "viewer")
|
||||
const acmeProjects: readonly (readonly [string, string, string])[] = [
|
||||
["documents", PROJECT.slug, PROJECT.name], // kit:documents
|
||||
["notes", "notes", "Team notes"], // kit:notes
|
||||
]
|
||||
for (const [moduleKey, slug, name] of acmeProjects) await project(ada, acme.slug, moduleKey, slug, name)
|
||||
|
||||
// Northwind: Grace owns it and gave it its own brand; Ada administers it, so she belongs to two
|
||||
// organizations and the switcher has something to switch between.
|
||||
await organization(grace, northwind)
|
||||
await member(grace, northwind.slug, ada, "admin")
|
||||
const northwindProjects: readonly (readonly [string, string, string])[] = [
|
||||
["documents", PROJECT.slug, PROJECT.name], // kit:documents
|
||||
]
|
||||
for (const [moduleKey, slug, name] of northwindProjects) await project(grace, northwind.slug, moduleKey, slug, name)
|
||||
const branded = (await as(grace).callByName("branding.brand.get", { org: northwind.slug })) as {
|
||||
displayName: string | null
|
||||
}
|
||||
if (branded.displayName === null) {
|
||||
await as(grace).callByName("branding.brand.set", { org: northwind.slug, ...northwind.brand })
|
||||
created.push(`brand of ${northwind.slug}`)
|
||||
}
|
||||
|
||||
// kit:notes:begin
|
||||
// A few notes in Acme's notes project, so a module's pages have something on them.
|
||||
const notes = (await as(ada).callByName("notes.note.list", {
|
||||
org: acme.slug,
|
||||
workspace: WORKSPACE,
|
||||
project: "notes",
|
||||
})) as { notes: unknown[] }
|
||||
if (notes.notes.length === 0) {
|
||||
for (const [title, body] of [
|
||||
["Welcome", "These notes are demo data. Add one, attach a file, delete it."],
|
||||
["Roles", "Ada owns this organization, Grace is a member and Linus can only read."],
|
||||
] as const) {
|
||||
await as(ada).callByName("notes.note.create", {
|
||||
org: acme.slug,
|
||||
workspace: WORKSPACE,
|
||||
project: "notes",
|
||||
title,
|
||||
body,
|
||||
})
|
||||
}
|
||||
created.push("two notes")
|
||||
}
|
||||
// kit:notes:end
|
||||
|
||||
return {
|
||||
created,
|
||||
people: DEMO_PEOPLE,
|
||||
organizations: [acme, northwind].map(({ slug, name }) => ({ slug, name })),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": false,
|
||||
"declaration": false,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"//": "The emit build resolves a workspace package through node_modules to its dist, not through the source paths of tsconfig.paths.json: pnpm builds the packages in dependency order, so the dist is there."
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"//": "The worker typechecks against the modules' source (tsconfig.paths.json), and modules/index.ts names the modules' pages, which are React: so the worker program needs the JSX setting and the DOM library to read them, though it never renders one.",
|
||||
"extends": "../../tsconfig.paths.json",
|
||||
"compilerOptions": {
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"jsx": "react-jsx",
|
||||
"lib": ["ES2023", "dom"],
|
||||
"types": ["node"],
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in New Issue
Block a user