export type RetrievalSource =
  | "knowledgeStore"
  | "searchIndex"
  | "codeGraph"
  | "openApiRegistry"
  | "existingDocs"
  | "examples"
  | "tests"
  | "gitDiff"
  | "userProvided";

export type RetrievalRequest = {
  objective: string;
  target?: GenerationTarget;
  queryHints: string[];
  contextPolicy: ContextPolicy;
  outputPolicy: OutputPolicy;
  changedFiles?: string[];
};

export type RetrievalResult = {
  evidencePack: EvidencePack;
  diagnostics: Diagnostic[];
  stats: RetrievalStats;
};

export type RetrievalStats = {
  candidatesFound: number;
  candidatesAfterFiltering: number;
  itemsSelected: number;
  estimatedTokens: number;
  sourcesUsed: Record<RetrievalSource, number>;
};

export type RetrievalQueryPlan = {
  intents: RetrievalIntent[];
  exactLookups: ExactLookup[];
  keywordQueries: string[];
  graphSeeds: GraphNodeRef[];
  requiredSources: RetrievalSource[];
  optionalSources: RetrievalSource[];
};

export type RetrievalIntent =
  | "writeGuide"
  | "writeReference"
  | "updateFromDiff"
  | "reviewDocs"
  | "explainConcept"
  | "generateTroubleshooting"
  | "documentApiOperation"
  | "documentCliCommand"
  | "documentConfig";

Generate a guide for docforge build pipeline

{
  "intents": ["writeGuide"],
  "keywordQueries": ["build pipeline", "docforge build", "static site build"],
  "exactLookups": [
    { "type": "cliCommand", "name": "docforge build" }
  ],
  "graphSeeds": [
    { "type": "semanticArtifact", "id": "cli:docforge-build" }
  ],
  "requiredSources": ["knowledgeStore", "codeGraph", "existingDocs"]
}

{ type: "apiOperation", operationKey: "public:POST /users" }

export type RetrievalCandidate = {
  id: string;
  source: RetrievalSource;
  kind: EvidenceItem["kind"];
  title: string;
  content: string;
  provenance: ProvenanceRef[];
  score: number;
  confidence: Confidence;
  sensitivity: SensitivityLevel;
  metadata: Record<string, unknown>;
};

export async function exactLookup(
  plan: RetrievalQueryPlan,
  store: KnowledgeStore
): Promise<RetrievalCandidate[]> {
  const candidates: RetrievalCandidate[] = [];

  for (const lookup of plan.exactLookups) {
    if (lookup.type === "cliCommand") {
      candidates.push(...await lookupCliCommand(store, lookup.name));
    }

    if (lookup.type === "apiOperation") {
      candidates.push(...await lookupApiOperation(store, lookup.operationKey));
    }

    if (lookup.type === "configField") {
      candidates.push(...await lookupConfigField(store, lookup.path));
    }
  }

  return candidates;
}

CREATE VIRTUAL TABLE retrieval_fts
USING fts5(
  title,
  content,
  kind,
  path,
  content='retrieval_documents',
  content_rowid='rowid'
);

finalScore =
  exactScore * 2.0 +
  keywordScore * 1.0 +
  semanticScore * 0.8 +
  graphScore * 1.2 +
  freshnessBoost +
  publicSurfaceBoost -
  sensitivityPenalty

cli:docforge-build

cli command -> handler symbol
handler symbol -> calls buildSite
handler symbol -> references BuildConfig
cli command -> documented by docs/reference/cli-build
handler symbol -> tested by build-command.test.ts

export type RetrievalGraphExpansionConfig = {
  maxDepth: number;
  maxNodes: number;
  relationWeights: Partial<Record<RelationKind, number>>;
};

export async function retrieveFromGraph(
  seeds: GraphNodeRef[],
  store: KnowledgeStore,
  config: RetrievalGraphExpansionConfig
): Promise<RetrievalCandidate[]> {
  const candidates: RetrievalCandidate[] = [];
  const visited = new Set<string>();
  const queue = seeds.map((seed) => ({
    ref: seed,
    depth: 0,
    score: 1.0,
  }));

  while (queue.length > 0 && visited.size < config.maxNodes) {
    const item = queue.shift()!;
    const key = graphRefKey(item.ref);

    if (visited.has(key) || item.depth > config.maxDepth) {
      continue;
    }

    visited.add(key);

    const candidate = await graphNodeToCandidate(store, item.ref, item.score);
    if (candidate) candidates.push(candidate);

    const edges = await store.graph.outgoing(item.ref);

    for (const edge of edges) {
      const weight = config.relationWeights[edge.kind] ?? 0;
      if (weight <= 0) continue;

      queue.push({
        ref: edge.to,
        depth: item.depth + 1,
        score: item.score * weight * confidenceWeight(edge.confidence),
      });
    }
  }

  return candidates;
}

docPage --documents--> target

export type RelationExpansionRule = {
  kind: RelationKind;
  direction: "outgoing" | "incoming" | "both";
  weight: number;
};

const DEFAULT_EXPANSION_RULES: RelationExpansionRule[] = [
  { kind: "documents", direction: "incoming", weight: 1.0 },
  { kind: "definesCliCommand", direction: "incoming", weight: 1.0 },
  { kind: "handlesRoute", direction: "incoming", weight: 1.0 },
  { kind: "definesConfigField", direction: "incoming", weight: 1.0 },
  { kind: "tests", direction: "incoming", weight: 0.8 },
  { kind: "exampleOf", direction: "incoming", weight: 0.8 },
  { kind: "calls", direction: "outgoing", weight: 0.4 },
  { kind: "references", direction: "outgoing", weight: 0.5 },
];

changedFiles: ["src/commands/build.ts"]

export async function retrieveForDiffUpdate(
  changedFiles: string[],
  store: KnowledgeStore
): Promise<RetrievalCandidate[]> {
  const candidates: RetrievalCandidate[] = [];

  for (const path of changedFiles) {
    const artifact = await store.artifacts.getByPath(path);
    if (!artifact) continue;

    candidates.push(artifactToCandidate(artifact));

    const symbols = await store.symbols.listByArtifact(artifact.id);
    candidates.push(...symbols.map(symbolToCandidate));

    const impact = await impactForChangedPath(store, path);
    for (const page of impact.affectedPages) {
      candidates.push(await docPageToCandidate(store, page.pageId));
    }
  }

  return candidates;
}

[EVIDENCE docs:reference-cli-build]
Kind: docPage
Source: docs/reference/cli-build.mdx
Route: /reference/cli-build

Existing docs content:
...

it("fails build when internal links are broken", async () => ...)

[EVIDENCE test:build-fails-on-broken-links]
Kind: test
Source: tests/build.test.ts:34-58
Confidence: medium

Behavior:
Build fails when internal links are broken.

[EVIDENCE example:basic-config]
Kind: example
Source: examples/basic/docforge.config.json
Confidence: high

Example config:
{
  "site": { "title": "Acme Docs" },
  "docs": { "root": "docs" }
}

export type SensitivityLevel =
  | "public"
  | "internal"
  | "secretLike"
  | "excluded";

export function filterBySensitivity(
  candidates: RetrievalCandidate[],
  policy: ContextPolicy
): RetrievalCandidate[] {
  return candidates.filter((candidate) => {
    if (candidate.sensitivity === "excluded") return false;
    if (candidate.sensitivity === "secretLike") return false;
    if (candidate.sensitivity === "internal" && !policy.sensitivity.allowInternal) return false;
    return true;
  });
}

export type TokenBudget = {
  maxTokens: number;
  reservedForInstructions: number;
  reservedForOutput: number;
  availableForEvidence: number;
};

export function selectWithinTokenBudget(
  candidates: RetrievalCandidate[],
  budget: TokenBudget
): EvidenceItem[] {
  const selected: EvidenceItem[] = [];
  let used = 0;

  for (const candidate of candidates.sort(compareCandidateRank)) {
    const item = candidateToEvidenceItem(candidate);
    const tokens = estimateTokens(item.content);

    if (used + tokens > budget.availableForEvidence) {
      const compressed = compressEvidenceItem(item, budget.availableForEvidence - used);

      if (compressed) {
        selected.push(compressed);
        used += estimateTokens(compressed.content);
      }

      continue;
    }

    selected.push(item);
    used += tokens;
  }

  return selected;
}

export type CodeEvidenceSlice = {
  symbolId: SymbolId;
  signature: string;
  docComment?: string;
  sourceExcerpt?: string;
  dependencies?: string[];
};

export function codeSymbolToEvidence(
  symbol: CodeSymbol,
  source: string,
  policy: ContextPolicy
): EvidenceItem {
  const content = [
    `Qualified name: ${symbol.qualifiedName}`,
    `Signature: ${symbol.signature ?? symbol.displayName}`,
    symbol.docComment ? `Doc comment:\n${symbol.docComment}` : undefined,
    policy.includeSourceCode ? excerptSymbolSource(symbol, source) : undefined,
  ].filter(Boolean).join("\n\n");

  return {
    id: `symbol:${symbol.id}`,
    kind: "codeSymbol",
    title: symbol.displayName,
    content,
    provenance: [provenanceFromSymbol(symbol)],
    confidence: "high",
    priority: 0.8,
  };
}

export function dedupeCandidates(candidates: RetrievalCandidate[]): RetrievalCandidate[] {
  const byCanonical = new Map<string, RetrievalCandidate>();

  for (const candidate of candidates) {
    const key = canonicalCandidateKey(candidate);
    const existing = byCanonical.get(key);

    if (!existing || candidate.score > existing.score) {
      byCanonical.set(key, mergeCandidate(existing, candidate));
    }
  }

  return [...byCanonical.values()];
}

export type RankingFeatures = {
  exactMatch: number;
  keywordScore: number;
  semanticScore: number;
  graphScore: number;
  confidence: number;
  publicSurface: number;
  freshness: number;
  sourceAuthority: number;
  targetRelevance: number;
  sensitivityPenalty: number;
};

export function rankCandidate(candidate: RetrievalCandidate): number {
  const f = candidate.features;

  return (
    f.exactMatch * 3.0 +
    f.sourceAuthority * 2.0 +
    f.graphScore * 1.5 +
    f.keywordScore * 1.0 +
    f.semanticScore * 0.8 +
    f.confidence * 1.2 +
    f.publicSurface * 0.8 +
    f.freshness * 0.5 -
    f.sensitivityPenalty * 5.0
  );
}

export function freshnessScore(candidate: RetrievalCandidate): number {
  if (candidate.metadata.stale === true) return 0.1;
  if (candidate.metadata.sourceHashMatches === true) return 1.0;
  return 0.5;
}

export type MissingEvidence = {
  topic: string;
  reason: string;
  impact: "low" | "medium" | "high";
};

export type EvidencePack = {
  id: string;
  objective: string;
  items: EvidenceItem[];
  missingEvidence: MissingEvidence[];
  constraints: string[];
  stats: {
    tokenEstimate: number;
    sourceCounts: Record<string, number>;
  };
};

# Evidence Pack

Objective:
Generate a guide for the static build pipeline.

Rules:
- Use only evidence below.
- Cite evidence IDs for factual claims.
- If evidence is missing, say so.

Missing evidence:
- Deployment hosting adapter details are not available.

Evidence:
...

cli:docforge-build
symbol:buildSite
openapi:public:createUser
config:build.outputDir
doc:static-build-pipeline
test:build-fails-on-broken-links
example:basic-config

Plan a guide for API reference generation.

Write the Quickstart page for DocForge CLI.

Update only managed regions or propose patch for affected docs.
Do not rewrite unrelated sections.

{
  code: "retrieval.sensitiveEvidenceExcluded",
  severity: "warning",
  category: "ai",
  message: "Some relevant evidence was excluded because it is marked internal or secret-like.",
  hint: "Adjust context policy only if this is a trusted generation workflow.",
}

"build"

warning retrieval.query.tooBroad
The retrieval query "build" matched many unrelated artifacts.

Hint:
Specify a target such as "docforge build command" or "static site build pipeline".

export const CONTEXT_PRESETS = {
  publicDocs: {
    includeSourceCode: false,
    includeTests: true,
    includeExamples: true,
    includeExistingDocs: true,
    sensitivity: { allowInternal: false, allowSecretLike: false },
  },
  internalArchitectureDocs: {
    includeSourceCode: true,
    includeTests: true,
    includeExamples: true,
    includeExistingDocs: true,
    sensitivity: { allowInternal: true, allowSecretLike: false },
  },
  apiReference: {
    includeOpenApi: true,
    includeSourceCode: false,
    includeTests: false,
    includeExamples: true,
    sensitivity: { allowInternal: false, allowSecretLike: false },
  },
};

export type RetrievalCacheKey = {
  requestHash: string;
  storeSnapshotHash: string;
  contextPolicyHash: string;
  retrieverVersion: string;
};

.docforge/reports/generation/<job-id>/evidence-pack.json

export type RetrievalEvalCase = {
  id: string;
  request: RetrievalRequest;
  expectedEvidenceIds: string[];
  forbiddenEvidenceIds: string[];
  maxTokenBudget: number;
};

{
  "id": "cli-build-guide",
  "request": {
    "objective": "Write a guide for docforge build command"
  },
  "expectedEvidenceIds": [
    "cli:docforge-build",
    "config:build.outputDir",
    "doc:static-site-build-pipeline"
  ],
  "forbiddenEvidenceIds": [
    "api:public:createUser"
  ]
}

it("retrieves CLI command evidence for build guide", async () => {
  const store = seedStoreWithCliCommand();
  const result = await retrieveContext({
    objective: "Write a guide for docforge build",
    target: { type: "cliCommand", commandId: "cli:docforge-build" },
    contextPolicy: defaultPolicy,
  }, store);

  expect(result.evidencePack.items.map((i) => i.id)).toContain("cli:docforge-build");
});

it("excludes secret-like evidence", async () => {
  const result = await retrieveContext(request, storeWithSecretArtifact);

  expect(result.evidencePack.items.some((i) => i.id === "secret:env")).toBe(false);
});

const evidence = await retriever.retrieve(request);
const prompt = promptBuilder.build(request, evidence);

promptBuilder.lookupMoreContext(...)

export type RetrievalRepository = {
  findSemanticArtifacts(query: SemanticArtifactQuery): Promise<RetrievalCandidate[]>;
  findSymbols(query: SymbolQuery): Promise<RetrievalCandidate[]>;
  findDocPages(query: DocPageQuery): Promise<RetrievalCandidate[]>;
  findExamples(query: ExampleQuery): Promise<RetrievalCandidate[]>;
  findTests(query: TestQuery): Promise<RetrievalCandidate[]>;
};

search hit -> page/section -> doc page candidate -> maybe graph mappings -> formal artifacts

config field build.outputDir
<- documents - docs/reference/configuration
<- references - build command handler
<- exampleOf - examples/basic-config

packages/retrieval/
  src/
    request.ts
    query-plan.ts
    candidate.ts
    exact-lookup.ts
    keyword.ts
    semantic.ts
    graph-expand.ts
    diff.ts
    ranking.ts
    dedupe.ts
    sensitivity.ts
    token-budget.ts
    compression.ts
    evidence-pack.ts
    diagnostics.ts
    eval.ts
    __tests__/
      cli-command-retrieval.test.ts
      api-operation-retrieval.test.ts
      config-retrieval.test.ts
      sensitivity.test.ts
      token-budget.test.ts