Discord
diff --git a/packages/console/app/src/routes/index.tsx b/packages/console/app/src/routes/index.tsx
index 281cec099a..a9648fc769 100644
--- a/packages/console/app/src/routes/index.tsx
+++ b/packages/console/app/src/routes/index.tsx
@@ -67,7 +67,7 @@ export default function Home() {
Discord
diff --git a/packages/opencode/AGENTS.md b/packages/opencode/AGENTS.md
index 287cbc2658..099bd6f70c 100644
--- a/packages/opencode/AGENTS.md
+++ b/packages/opencode/AGENTS.md
@@ -20,6 +20,9 @@
## Architecture
- **Tools**: Implement `Tool.Info` interface with `execute()` method
+- **Metadata**: Use `Tool.define
` and keep `ctx.metadata({ metadata })` payloads JSON-safe
+- **Plugins**: Custom tool hooks may return a string or `{ output, title?, metadata? }` which is forwarded directly to the agent
+- **Telemetry**: Wrap tool execution in `measure()` to emit duration logs; resolve file paths via `guard()` before touching the filesystem. See `docs/tool-authoring.md` for patterns.
- **Context**: Pass `sessionID` in tool context, use `App.provide()` for DI
- **Validation**: All inputs validated with Zod schemas
- **Logging**: Use `Log.create({ service: "name" })` pattern
diff --git a/packages/opencode/docs/evaluation-integration.md b/packages/opencode/docs/evaluation-integration.md
new file mode 100644
index 0000000000..0f76efc90e
--- /dev/null
+++ b/packages/opencode/docs/evaluation-integration.md
@@ -0,0 +1,587 @@
+# Evaluation Framework Integration Guide
+
+This guide shows how to deeply integrate the evaluation framework with baseline tracking, time-series analysis, and automatic monitoring.
+
+## Architecture Overview
+
+```
+┌─────────────┐
+│ Trace │ Completed
+│ Completion │──────────┐
+└─────────────┘ │
+ ▼
+ ┌──────────────────────┐
+ │ EvaluationIntegration│
+ │ (Auto-Processor) │
+ └──────────────────────┘
+ │
+ ┌────────────────┼────────────────┐
+ │ │ │
+ ▼ ▼ ▼
+┌──────────────┐ ┌──────────────┐ ┌──────────────┐
+│ Evaluation │ │ TimeSeries │ │ Baseline │
+│ Engine │ │ Tracking │ │ Comparison │
+└──────────────┘ └──────────────┘ └──────────────┘
+ │ │ │
+ └────────────────┼────────────────┘
+ ▼
+ ┌──────────────────────┐
+ │ Alert Generation │
+ │ (Regression/Anomaly)│
+ └──────────────────────┘
+ │
+ ▼
+ ┌──────────────────────┐
+ │ Notifications & │
+ │ Dashboard │
+ └──────────────────────┘
+```
+
+## Quick Start
+
+### 1. Define Metrics
+
+First, register the metrics you want to track:
+
+```typescript
+import { Metric } from "@opencode/evaluation"
+
+// Register metrics
+await Metric.register({
+ id: "error-rate",
+ name: "Error Rate",
+ description: "Percentage of tool calls that failed",
+ version: "1.0.0",
+ category: "reliability",
+ evaluator: { type: "heuristic", function: "toolErrorRate" },
+ higherIsBetter: false,
+ threshold: { pass: 0.05 }, // Max 5% error rate
+})
+
+await Metric.register({
+ id: "latency",
+ name: "Response Time",
+ description: "Total trace duration in milliseconds",
+ version: "1.0.0",
+ category: "performance",
+ evaluator: { type: "heuristic", function: "duration" },
+ higherIsBetter: false,
+ threshold: { pass: 5000 }, // Max 5 seconds
+})
+
+await Metric.register({
+ id: "cost",
+ name: "Total Cost",
+ description: "Sum of all LLM API costs",
+ version: "1.0.0",
+ category: "cost",
+ evaluator: { type: "heuristic", function: "totalCost" },
+ higherIsBetter: false,
+})
+```
+
+### 2. Create Baselines
+
+Establish performance baselines from production traces:
+
+```typescript
+import { Baseline } from "@opencode/evaluation"
+
+// Create production baseline
+const prodBaseline = await Baseline.create({
+ id: "prod-baseline-v1",
+ name: "Production Baseline v1",
+ description: "Reference performance from Oct 2024",
+ metricIDs: ["error-rate", "latency", "cost"],
+ minSampleSize: 20,
+ regressionThreshold: 0.15, // 15% degradation triggers alert
+ tags: ["production", "v1"],
+})
+
+// Add historical traces to baseline
+const historicalTraces = await Trace.list({
+ since: Date.now() - 7 * 24 * 60 * 60 * 1000, // Last 7 days
+ hasErrors: false, // Only successful traces
+})
+
+for await (const trace of historicalTraces) {
+ await Baseline.addTrace(prodBaseline.id, trace)
+}
+
+console.log(`Baseline created with ${prodBaseline.traceIDs.length} traces`)
+```
+
+### 3. Enable Auto-Evaluation
+
+Set up automatic evaluation and monitoring:
+
+```typescript
+import { EvaluationIntegration } from "@opencode/evaluation"
+
+// Enable auto-evaluation
+await EvaluationIntegration.enableAutoEvaluation({
+ metricIDs: ["error-rate", "latency", "cost"],
+ recordTimeSeries: true, // Track trends over time
+ checkBaselines: true, // Compare against baseline
+ detectAnomalies: true, // Detect statistical outliers
+ anomalyThreshold: 3, // 3-sigma rule
+ tags: {
+ environment: "production",
+ version: "1.0.0",
+ },
+})
+
+console.log("Auto-evaluation enabled")
+```
+
+### 4. Set Up Alerts
+
+Register callbacks for different alert types:
+
+```typescript
+// Monitor regressions
+EvaluationIntegration.onRegression((alert) => {
+ console.error(`🔴 REGRESSION DETECTED`)
+ console.error(` Metric: ${alert.metricID}`)
+ console.error(` Trace: ${alert.traceID}`)
+ console.error(` Baseline: ${alert.baselineValue.toFixed(3)}`)
+ console.error(` Current: ${alert.currentValue.toFixed(3)}`)
+ console.error(` Change: ${alert.percentChange.toFixed(1)}%`)
+
+ // Send to alerting system
+ sendSlackAlert(`Regression in ${alert.metricID}: ${alert.percentChange.toFixed(1)}% worse`)
+ createJiraTicket(alert)
+})
+
+// Monitor anomalies
+EvaluationIntegration.onAnomaly((alert) => {
+ console.warn(`⚠️ ANOMALY DETECTED`)
+ console.warn(` Metric: ${alert.metricID}`)
+ console.warn(` Current: ${alert.currentValue.toFixed(3)}`)
+ console.warn(` Expected: ${alert.expectedRange.min.toFixed(3)} - ${alert.expectedRange.max.toFixed(3)}`)
+ console.warn(` Z-Score: ${alert.zScore.toFixed(2)}σ`)
+
+ // Log for investigation
+ logAnomalyForInvestigation(alert)
+})
+
+// Celebrate improvements
+EvaluationIntegration.onImprovement((alert) => {
+ console.log(`🎉 IMPROVEMENT DETECTED`)
+ console.log(` Metric: ${alert.metricID}`)
+ console.log(` Change: ${Math.abs(alert.percentChange).toFixed(1)}% better`)
+
+ // Track wins
+ recordMetricsImprovement(alert)
+})
+```
+
+## Advanced Usage
+
+### A/B Testing
+
+Compare two different agent configurations:
+
+```typescript
+// Create baseline for version A
+const baselineA = await Baseline.create({
+ id: "agent-v1-baseline",
+ name: "Agent v1 Baseline",
+ description: "Performance of original agent",
+ metricIDs: ["error-rate", "latency", "cost"],
+ minSampleSize: 30,
+ tags: ["v1"],
+})
+
+// Create baseline for version B
+const baselineB = await Baseline.create({
+ id: "agent-v2-baseline",
+ name: "Agent v2 Baseline",
+ description: "Performance with new prompt",
+ metricIDs: ["error-rate", "latency", "cost"],
+ minSampleSize: 30,
+ tags: ["v2"],
+})
+
+// Collect data for both versions...
+// (run production traffic through both)
+
+// Compare after sufficient samples
+const abResult = await Baseline.compareAB(baselineA.id, baselineB.id)
+
+console.log(`A/B Test Results`)
+console.log(` Overall Winner: ${abResult.overallWinner}`)
+console.log(` Sample Sizes: A=${abResult.sampleSizeA}, B=${abResult.sampleSizeB}`)
+console.log(`\nMetric Breakdown:`)
+
+for (const metric of abResult.metrics) {
+ console.log(` ${metric.metricID}:`)
+ console.log(` Winner: ${metric.winner}`)
+ console.log(` A: ${metric.meanA.toFixed(3)}, B: ${metric.meanB.toFixed(3)}`)
+ console.log(` Change: ${metric.percentChange.toFixed(1)}%`)
+ console.log(` Confidence: ${(metric.confidence * 100).toFixed(1)}%`)
+}
+
+// Roll out winner to 100% traffic
+if (abResult.overallWinner === "B") {
+ deployVersion("v2")
+}
+```
+
+### Trend Analysis
+
+Analyze performance trends over time:
+
+```typescript
+// Analyze error rate trend over last 30 days
+const errorTrend = await TimeSeries.analyzeTrend("error-rate", {
+ days: 30,
+ anomalyThreshold: 2, // 2-sigma for anomaly detection
+})
+
+console.log(`Error Rate Trend Analysis`)
+console.log(` Trend: ${errorTrend.trend}`) // "improving", "degrading", or "stable"
+console.log(` Strength: ${(errorTrend.trendStrength * 100).toFixed(1)}%`)
+console.log(` Slope: ${errorTrend.slope.toFixed(6)}/day`)
+console.log(` Overall Change: ${errorTrend.changePercent.toFixed(1)}%`)
+console.log(` Anomalies: ${errorTrend.anomalies.length}`)
+
+if (errorTrend.trend === "degrading") {
+ console.warn(`⚠️ Error rate has been degrading over last 30 days`)
+ investigateDegradation(errorTrend)
+}
+
+// Detect anomalies in real-time
+const currentErrorRate = 0.08 // 8%
+const anomalyCheck = await TimeSeries.detectAnomaly("error-rate", currentErrorRate, 14)
+
+if (anomalyCheck.isAnomaly) {
+ console.error(`Current error rate ${currentErrorRate} is anomalous!`)
+ console.error(` Expected range: ${anomalyCheck.expectedRange.min.toFixed(3)} - ${anomalyCheck.expectedRange.max.toFixed(3)}`)
+ console.error(` Historical mean: ${anomalyCheck.historicalMean.toFixed(3)}`)
+}
+```
+
+### Dashboard Integration
+
+Build a monitoring dashboard:
+
+```typescript
+// Get dashboard data for visualization
+const dashboard = await EvaluationIntegration.getDashboard({
+ since: Date.now() - 30 * 24 * 60 * 60 * 1000, // Last 30 days
+ metricIDs: ["error-rate", "latency", "cost"],
+ period: "day", // Daily aggregates
+})
+
+// Render dashboard
+for (const metric of dashboard.metrics) {
+ console.log(`\n${metric.metric.name}`)
+ console.log(` Data Points: ${metric.dataPoints}`)
+
+ if (metric.trend) {
+ console.log(` Trend: ${metric.trend.trend} (${(metric.trend.trendStrength * 100).toFixed(1)}%)`)
+ console.log(` 30-day Change: ${metric.trend.changePercent.toFixed(1)}%`)
+ }
+
+ console.log(` Last 10 Days:`)
+ for (const agg of metric.aggregates.slice(-10)) {
+ const date = new Date(agg.periodStart).toLocaleDateString()
+ console.log(` ${date}: ${agg.mean.toFixed(3)} (min: ${agg.min.toFixed(3)}, max: ${agg.max.toFixed(3)})`)
+ }
+
+ console.log(` Baselines:`)
+ for (const baseline of metric.baselines) {
+ if (baseline.statistics) {
+ console.log(` ${baseline.name}: ${baseline.statistics.mean.toFixed(3)} ± ${baseline.statistics.stdDev.toFixed(3)}`)
+ }
+ }
+}
+```
+
+### Historical Re-evaluation
+
+Re-evaluate old traces after updating metrics:
+
+```typescript
+// Get all traces from last month
+const traces = await Trace.list({
+ since: Date.now() - 30 * 24 * 60 * 60 * 1000,
+})
+
+const traceIDs = []
+for await (const trace of traces) {
+ traceIDs.push(trace.id)
+}
+
+console.log(`Re-evaluating ${traceIDs.length} historical traces`)
+
+// Batch evaluate with new metrics
+await EvaluationIntegration.evaluateTraces(traceIDs, {
+ metricIDs: ["new-metric-v2", "error-rate", "latency"],
+ recordTimeSeries: true,
+ checkBaselines: false, // Don't alert on historical data
+})
+
+console.log(`Historical evaluation complete`)
+```
+
+### Custom Alert Routing
+
+Route alerts to different channels based on severity:
+
+```typescript
+EvaluationIntegration.onAlert((alert) => {
+ // Route based on alert type and severity
+ switch (alert.type) {
+ case "regression":
+ if (Math.abs(alert.percentChange) > 50) {
+ // Critical regression
+ sendPagerDuty({
+ severity: "critical",
+ summary: `Critical regression in ${alert.metricID}`,
+ details: alert,
+ })
+ } else if (Math.abs(alert.percentChange) > 20) {
+ // Major regression
+ sendSlack({
+ channel: "#incidents",
+ text: `⚠️ Major regression in ${alert.metricID}: ${alert.percentChange.toFixed(1)}% worse`,
+ alert,
+ })
+ } else {
+ // Minor regression
+ sendSlack({
+ channel: "#metrics",
+ text: `Regression in ${alert.metricID}: ${alert.percentChange.toFixed(1)}% worse`,
+ alert,
+ })
+ }
+ break
+
+ case "anomaly":
+ if (Math.abs(alert.zScore) > 5) {
+ // Extreme anomaly
+ sendSlack({
+ channel: "#incidents",
+ text: `🔴 Extreme anomaly in ${alert.metricID}: ${alert.zScore.toFixed(1)}σ`,
+ alert,
+ })
+ } else {
+ // Normal anomaly
+ logToDatadog("anomaly_detected", alert)
+ }
+ break
+
+ case "improvement":
+ // Celebrate improvements
+ sendSlack({
+ channel: "#wins",
+ text: `🎉 Improvement in ${alert.metricID}: ${Math.abs(alert.percentChange).toFixed(1)}% better!`,
+ alert,
+ })
+ break
+ }
+})
+```
+
+## Best Practices
+
+### 1. Baseline Management
+
+- **Create separate baselines** for different environments (dev, staging, prod)
+- **Version your baselines** when making significant agent changes
+- **Maintain minimum sample sizes** (>20 traces) for statistical significance
+- **Update baselines regularly** to reflect expected performance
+
+```typescript
+// Environment-specific baselines
+await Baseline.create({
+ id: "prod-baseline",
+ tags: ["production", "us-east-1"],
+ minSampleSize: 50,
+ regressionThreshold: 0.10, // Strict for prod
+})
+
+await Baseline.create({
+ id: "staging-baseline",
+ tags: ["staging"],
+ minSampleSize: 20,
+ regressionThreshold: 0.25, // More lenient for staging
+})
+```
+
+### 2. Metric Selection
+
+- **Start with core metrics**: error rate, latency, cost
+- **Add domain-specific metrics** gradually
+- **Avoid metric overload**: 5-10 key metrics is usually sufficient
+- **Group related metrics** using tags
+
+### 3. Alert Tuning
+
+- **Start with conservative thresholds** to avoid alert fatigue
+- **Adjust based on false positive rate**
+- **Use different thresholds** for different metrics
+- **Implement alert deduplication** for noisy metrics
+
+### 4. Time-Series Analysis
+
+- **Use appropriate time windows**:
+ - Anomaly detection: 7-14 days
+ - Trend analysis: 30-90 days
+- **Consider seasonality**: weekday vs weekend patterns
+- **Filter outliers** when establishing baselines
+
+### 5. Performance
+
+- **Batch historical evaluations** during off-peak hours
+- **Use tags** to filter time-series queries
+- **Archive old data** periodically
+- **Index frequently queried fields**
+
+## Integration with CI/CD
+
+### Pre-Deployment Checks
+
+```typescript
+// In CI/CD pipeline, before deployment
+async function preDeploymentCheck() {
+ // Evaluate test traces against new code
+ const testTraces = await runIntegrationTests()
+
+ for (const trace of testTraces) {
+ await EvaluationIntegration.evaluateTrace(trace.id, {
+ metricIDs: ["error-rate", "latency", "cost"],
+ checkBaselines: true,
+ })
+ }
+
+ // Check if any regressions detected
+ let hasRegressions = false
+
+ const unsubscribe = EvaluationIntegration.onRegression((alert) => {
+ console.error(`Blocking deployment: regression in ${alert.metricID}`)
+ hasRegressions = true
+ })
+
+ // Wait for async processing
+ await new Promise(resolve => setTimeout(resolve, 1000))
+
+ unsubscribe()
+
+ if (hasRegressions) {
+ throw new Error("Deployment blocked due to regressions")
+ }
+
+ console.log("✅ No regressions detected, proceeding with deployment")
+}
+```
+
+### Post-Deployment Monitoring
+
+```typescript
+// Monitor for 1 hour after deployment
+async function postDeploymentMonitor(deploymentID: string) {
+ console.log(`Monitoring deployment ${deploymentID}`)
+
+ const alerts: Alert[] = []
+ const unsubscribe = EvaluationIntegration.onAlert((alert) => {
+ alerts.push(alert)
+ })
+
+ // Wait 1 hour
+ await new Promise(resolve => setTimeout(resolve, 60 * 60 * 1000))
+
+ unsubscribe()
+
+ // Check alert counts
+ const regressions = alerts.filter(a => a.type === "regression")
+ const anomalies = alerts.filter(a => a.type === "anomaly")
+
+ if (regressions.length > 5 || anomalies.length > 10) {
+ console.error(`Deployment ${deploymentID} showing issues, consider rollback`)
+ console.error(` Regressions: ${regressions.length}`)
+ console.error(` Anomalies: ${anomalies.length}`)
+
+ return { healthy: false, alerts }
+ }
+
+ console.log(`✅ Deployment ${deploymentID} healthy`)
+ return { healthy: true, alerts }
+}
+```
+
+## Troubleshooting
+
+### No Alerts Being Generated
+
+1. Check auto-evaluation is enabled:
+```typescript
+EvaluationIntegration.disableAutoEvaluation()
+await EvaluationIntegration.enableAutoEvaluation({ /* config */ })
+```
+
+2. Verify metrics are registered:
+```typescript
+const metric = await Metric.get("your-metric-id")
+console.log(metric)
+```
+
+3. Check baseline sample sizes:
+```typescript
+const baseline = await Baseline.get("your-baseline-id")
+console.log(`Sample size: ${baseline.traceIDs.length} (min: ${baseline.minSampleSize})`)
+```
+
+### Too Many False Positive Alerts
+
+1. Increase regression threshold:
+```typescript
+await Baseline.update("baseline-id", {
+ regressionThreshold: 0.25, // From 0.15 to 0.25
+})
+```
+
+2. Increase anomaly threshold:
+```typescript
+await EvaluationIntegration.enableAutoEvaluation({
+ // ... other config
+ anomalyThreshold: 4, // From 3 to 4 sigma
+})
+```
+
+3. Increase baseline sample size:
+```typescript
+await Baseline.update("baseline-id", {
+ minSampleSize: 50, // From 20 to 50
+})
+```
+
+### Missing Time-Series Data
+
+1. Verify recording is enabled:
+```typescript
+await EvaluationIntegration.enableAutoEvaluation({
+ // ... other config
+ recordTimeSeries: true,
+})
+```
+
+2. Check for evaluation errors:
+```typescript
+// Look for error logs in evaluation engine
+```
+
+3. Manually record test data:
+```typescript
+const trace = await Trace.get("trace-id")
+await TimeSeries.record("metric-id", trace, { tag: "test" })
+```
+
+## Next Steps
+
+- Explore [Metric Definitions](./metrics.md) for creating custom metrics
+- Learn about [Heuristic Functions](./heuristics.md) for built-in evaluators
+- See [Dataset Testing](./datasets.md) for test suite management
+- Review [API Reference](./api-reference.md) for detailed documentation
diff --git a/packages/opencode/package.json b/packages/opencode/package.json
index f8d71b4ab3..2b28f3cde0 100644
--- a/packages/opencode/package.json
+++ b/packages/opencode/package.json
@@ -33,6 +33,7 @@
"@hono/standard-validator": "0.1.5",
"@hono/zod-validator": "catalog:",
"@modelcontextprotocol/sdk": "1.15.1",
+ "@octokit/rest": "22.0.0",
"@openauthjs/openauth": "0.4.3",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/sdk": "workspace:*",
@@ -42,12 +43,14 @@
"chokidar": "4.0.3",
"decimal.js": "10.5.0",
"diff": "8.0.2",
+ "exa-js": "1.9.3",
"fuzzysort": "3.1.0",
"gray-matter": "4.0.3",
"hono": "catalog:",
"hono-openapi": "1.0.7",
"ignore": "7.0.5",
"jsonc-parser": "3.3.1",
+ "linkedom": "0.18.12",
"minimatch": "10.0.3",
"open": "10.1.2",
"remeda": "catalog:",
diff --git a/packages/opencode/script/publish.ts b/packages/opencode/script/publish.ts
index 996ec46e74..7698e25e39 100755
--- a/packages/opencode/script/publish.ts
+++ b/packages/opencode/script/publish.ts
@@ -70,17 +70,17 @@ if (!snapshot) {
"options=('!debug' '!strip')",
"pkgrel=1",
"pkgdesc='The AI coding agent built for the terminal.'",
- "url='https://github.com/sst/opencode'",
+ "url='https://github.com/evalops/opencode'",
"arch=('aarch64' 'x86_64')",
"license=('MIT')",
"provides=('opencode')",
"conflicts=('opencode')",
"depends=('fzf' 'ripgrep')",
"",
- `source_aarch64=("\${pkgname}_\${pkgver}_aarch64.zip::https://github.com/sst/opencode/releases/download/v${version}/opencode-linux-arm64.zip")`,
+ `source_aarch64=("\${pkgname}_\${pkgver}_aarch64.zip::https://github.com/evalops/opencode/releases/download/v${version}/opencode-linux-arm64.zip")`,
`sha256sums_aarch64=('${arm64Sha}')`,
"",
- `source_x86_64=("\${pkgname}_\${pkgver}_x86_64.zip::https://github.com/sst/opencode/releases/download/v${version}/opencode-linux-x64.zip")`,
+ `source_x86_64=("\${pkgname}_\${pkgver}_x86_64.zip::https://github.com/evalops/opencode/releases/download/v${version}/opencode-linux-x64.zip")`,
`sha256sums_x86_64=('${x64Sha}')`,
"",
"package() {",
@@ -99,7 +99,7 @@ if (!snapshot) {
"options=('!debug' '!strip')",
"pkgrel=1",
"pkgdesc='The AI coding agent built for the terminal.'",
- "url='https://github.com/sst/opencode'",
+ "url='https://github.com/evalops/opencode'",
"arch=('aarch64' 'x86_64')",
"license=('MIT')",
"provides=('opencode')",
@@ -107,7 +107,7 @@ if (!snapshot) {
"depends=('fzf' 'ripgrep')",
"makedepends=('git' 'bun-bin' 'go')",
"",
- `source=("opencode-\${pkgver}.tar.gz::https://github.com/sst/opencode/archive/v${version}.tar.gz")`,
+ `source=("opencode-\${pkgver}.tar.gz::https://github.com/evalops/opencode/archive/v${version}.tar.gz")`,
`sha256sums=('SKIP')`,
"",
"build() {",
@@ -155,12 +155,12 @@ if (!snapshot) {
"# This file was generated by GoReleaser. DO NOT EDIT.",
"class Opencode < Formula",
` desc "The AI coding agent built for the terminal."`,
- ` homepage "https://github.com/sst/opencode"`,
+ ` homepage "https://github.com/evalops/opencode"`,
` version "${version.split("-")[0]}"`,
"",
" on_macos do",
" if Hardware::CPU.intel?",
- ` url "https://github.com/sst/opencode/releases/download/v${version}/opencode-darwin-x64.zip"`,
+ ` url "https://github.com/evalops/opencode/releases/download/v${version}/opencode-darwin-x64.zip"`,
` sha256 "${macX64Sha}"`,
"",
" def install",
@@ -168,7 +168,7 @@ if (!snapshot) {
" end",
" end",
" if Hardware::CPU.arm?",
- ` url "https://github.com/sst/opencode/releases/download/v${version}/opencode-darwin-arm64.zip"`,
+ ` url "https://github.com/evalops/opencode/releases/download/v${version}/opencode-darwin-arm64.zip"`,
` sha256 "${macArm64Sha}"`,
"",
" def install",
@@ -179,14 +179,14 @@ if (!snapshot) {
"",
" on_linux do",
" if Hardware::CPU.intel? and Hardware::CPU.is_64_bit?",
- ` url "https://github.com/sst/opencode/releases/download/v${version}/opencode-linux-x64.zip"`,
+ ` url "https://github.com/evalops/opencode/releases/download/v${version}/opencode-linux-x64.zip"`,
` sha256 "${x64Sha}"`,
" def install",
' bin.install "opencode"',
" end",
" end",
" if Hardware::CPU.arm? and Hardware::CPU.is_64_bit?",
- ` url "https://github.com/sst/opencode/releases/download/v${version}/opencode-linux-arm64.zip"`,
+ ` url "https://github.com/evalops/opencode/releases/download/v${version}/opencode-linux-arm64.zip"`,
` sha256 "${arm64Sha}"`,
" def install",
' bin.install "opencode"',
diff --git a/packages/opencode/src/agent/agent.ts b/packages/opencode/src/agent/agent.ts
index 252c0bd6bd..b3f70cb4e0 100644
--- a/packages/opencode/src/agent/agent.ts
+++ b/packages/opencode/src/agent/agent.ts
@@ -20,6 +20,8 @@ export namespace Agent {
edit: Config.Permission,
bash: z.record(z.string(), Config.Permission),
webfetch: Config.Permission.optional(),
+ fetchurl: Config.Permission.optional(),
+ websearch: Config.Permission.optional(),
}),
model: z
.object({
@@ -45,6 +47,8 @@ export namespace Agent {
"*": "allow",
},
webfetch: "allow",
+ fetchurl: "allow",
+ websearch: "allow",
}
const agentPermission = mergeAgentPermissions(defaultPermission, cfg.permission ?? {})
@@ -53,6 +57,8 @@ export namespace Agent {
edit: "deny",
bash: "ask",
webfetch: "allow",
+ fetchurl: "allow",
+ websearch: "allow",
},
cfg.permission ?? {},
)
diff --git a/packages/opencode/src/bus/index.ts b/packages/opencode/src/bus/index.ts
index 7fbefba449..53344404c9 100644
--- a/packages/opencode/src/bus/index.ts
+++ b/packages/opencode/src/bus/index.ts
@@ -2,6 +2,7 @@ import z from "zod/v4"
import type { ZodType } from "zod/v4"
import { Log } from "../util/log"
import { Instance } from "../project/instance"
+import { Context } from "../util/context"
export namespace Bus {
const log = Log.create({ service: "bus" })
@@ -15,6 +16,17 @@ export namespace Bus {
}
})
+ function getState() {
+ try {
+ return state()
+ } catch (error) {
+ if (error instanceof Context.NotFound && error.name === "instance") {
+ return null
+ }
+ throw error
+ }
+ }
+
export type EventDefinition = ReturnType
const registry = new Map()
@@ -51,6 +63,14 @@ export namespace Bus {
def: Definition,
properties: z.output,
) {
+ const currentState = getState()
+ if (!currentState) {
+ log.debug("skipping publish (no instance context)", {
+ type: def.type,
+ })
+ return
+ }
+
const payload = {
type: def.type,
properties,
@@ -60,7 +80,7 @@ export namespace Bus {
})
const pending = []
for (const key of [def.type, "*"]) {
- const match = state().subscriptions.get(key)
+ const match = currentState.subscriptions.get(key)
for (const sub of match ?? []) {
pending.push(sub(payload))
}
@@ -92,8 +112,14 @@ export namespace Bus {
}
function raw(type: string, callback: (event: any) => void) {
+ const currentState = getState()
+ if (!currentState) {
+ log.debug("skipping subscription (no instance context)", { type })
+ return () => {}
+ }
+
log.info("subscribing", { type })
- const subscriptions = state().subscriptions
+ const subscriptions = currentState.subscriptions
let match = subscriptions.get(type) ?? []
match.push(callback)
subscriptions.set(type, match)
diff --git a/packages/opencode/src/cli/cmd/github.ts b/packages/opencode/src/cli/cmd/github.ts
index e15243e769..f19cdb9369 100644
--- a/packages/opencode/src/cli/cmd/github.ts
+++ b/packages/opencode/src/cli/cmd/github.ts
@@ -78,12 +78,12 @@ export const GithubInstallCommand = cmd({
.text()
.then((text) => text.trim())
// match https or git pattern
- // ie. https://github.com/sst/opencode.git
- // ie. https://github.com/sst/opencode
- // ie. git@github.com:sst/opencode.git
- // ie. git@github.com:sst/opencode
- // ie. ssh://git@github.com/sst/opencode.git
- // ie. ssh://git@github.com/sst/opencode
+ // ie. https://github.com/evalops/opencode.git
+ // ie. https://github.com/evalops/opencode
+ // ie. git@github.com:evalops/opencode.git
+ // ie. git@github.com:evalops/opencode
+ // ie. ssh://git@github.com/evalops/opencode.git
+ // ie. ssh://git@github.com/evalops/opencode
const parsed = info.match(/^(?:(?:https?|ssh):\/\/)?(?:git@)?github\.com[:/]([^/]+)\/([^/.]+?)(?:\.git)?$/)
if (!parsed) {
prompts.log.error(`Could not find git repository. Please run this command from a git repository.`)
@@ -231,7 +231,7 @@ jobs:
uses: actions/checkout@v4
- name: Run opencode
- uses: sst/opencode/github@latest${envStr}
+ uses: evalops/opencode/github@latest${envStr}
with:
model: ${provider}/${model}
`.trim(),
diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts
index e04ed81036..b39367afb8 100644
--- a/packages/opencode/src/cli/cmd/run.ts
+++ b/packages/opencode/src/cli/cmd/run.ts
@@ -12,6 +12,7 @@ import { Identifier } from "../../id/id"
import { Agent } from "../../agent/agent"
import { Command } from "../../command"
import { SessionPrompt } from "../../session/prompt"
+import { ToolTelemetry } from "../../tool/telemetry"
const TOOL: Record = {
todowrite: ["Todo", UI.Style.TEXT_WARNING_BOLD],
@@ -150,6 +151,11 @@ export const RunCommand = cmd({
)
}
+ function formatDuration(duration: number) {
+ if (duration < 1000) return `${duration.toFixed(0)}ms`
+ return `${(duration / 1000).toFixed(2)}s`
+ }
+
function outputJsonEvent(type: string, data: any) {
if (args.format === "json") {
const jsonEvent = {
@@ -209,6 +215,19 @@ export const RunCommand = cmd({
}
})
+ Bus.subscribe(ToolTelemetry.Event.Sampled, async (evt) => {
+ const info = evt.properties
+ if (info.sessionID !== session.id) return
+ if (outputJsonEvent("tool_telemetry", { telemetry: info })) return
+ const [label, defaultColor] = TOOL[info.id] ?? [info.id, UI.Style.TEXT_INFO_BOLD]
+ const color = info.status === "success" ? defaultColor : UI.Style.TEXT_DANGER_BOLD
+ let title = `${label} ${formatDuration(info.duration)}`
+ if (info.status === "error" && info.error) {
+ title += ` – ${info.error}`
+ }
+ printEvent(color, "tele", title)
+ })
+
let errorMsg: string | undefined
Bus.subscribe(Session.Event.Error, async (evt) => {
const { sessionID, error } = evt.properties
diff --git a/packages/opencode/src/cli/cmd/stats.ts b/packages/opencode/src/cli/cmd/stats.ts
index 39ae86ba0b..ca7624ec12 100644
--- a/packages/opencode/src/cli/cmd/stats.ts
+++ b/packages/opencode/src/cli/cmd/stats.ts
@@ -1,4 +1,9 @@
import { cmd } from "./cmd"
+import { ToolHistory } from "../../tool/history"
+import type { TelemetryEvent } from "../../tool/telemetry-event"
+import { bootstrap } from "../bootstrap"
+import { Session } from "../../session"
+import { MessageV2 } from "../../session/message-v2"
interface SessionStats {
totalSessions: number
@@ -14,6 +19,7 @@ interface SessionStats {
}
}
toolUsage: Record
+ toolTelemetry: Record
dateRange: {
earliest: number
latest: number
@@ -22,11 +28,254 @@ interface SessionStats {
costPerDay: number
}
-export const StatsCommand = cmd({
+type ToolTelemetryStats = {
+ runs: number
+ errors: number
+ totalDuration: number
+ averageDuration: number
+ medianDuration: number
+ p95Duration: number
+ p99Duration: number
+ errorRate: number
+ successRate: number
+}
+
+type TelemetrySummary = {
+ windowStart?: number
+ windowEnd?: number
+ totalRuns: number
+ totalErrors: number
+ perDayErrorRate?: number
+ tools: Record
+}
+
+type DetailFormat = "pretty" | "ndjson" | "csv"
+
+type ToolComparison = {
+ tool: string
+ baseline?: ToolTelemetryStats
+ current?: ToolTelemetryStats
+}
+
+type TelemetryComparison = {
+ path: string
+ totalRunsDelta: number
+ totalErrorsDelta: number
+ toolComparisons: ToolComparison[]
+}
+
+type DetailOptions = {
+ format: DetailFormat
+ fields: string[]
+}
+
+type StatsArgs = {
+ json?: boolean
+ telemetry?: string
+ limit?: number
+ clear?: boolean
+ details?: boolean
+ detailsFormat?: DetailFormat
+ fields?: string
+ status?: string
+ since?: string
+ until?: string
+ compare?: string
+ warnLatency?: number
+ warnErrors?: number
+}
+
+export const StatsCommand = cmd({
command: "stats",
- handler: async () => {},
+ describe: "Show session and telemetry statistics",
+ builder: (yargs) =>
+ yargs
+ .option("json", {
+ describe: "Output raw JSON instead of formatted tables",
+ type: "boolean",
+ default: false,
+ })
+ .option("telemetry", {
+ describe: "Filter telemetry events by tool id (use 'all' for everything)",
+ type: "string",
+ })
+ .option("limit", {
+ describe: "Number of telemetry events to display",
+ type: "number",
+ default: 20,
+ })
+ .option("clear", {
+ describe: "Clear stored telemetry history before printing stats",
+ type: "boolean",
+ default: false,
+ })
+ .option("details", {
+ describe: "Print telemetry metadata for matching events",
+ type: "boolean",
+ default: false,
+ })
+ .option("details-format", {
+ describe: "Format for telemetry metadata output (pretty, ndjson, csv)",
+ type: "string",
+ choices: ["pretty", "ndjson", "csv"],
+ default: "pretty",
+ })
+ .option("fields", {
+ describe: "Comma separated metadata keys to include in details",
+ type: "string",
+ })
+ .option("status", {
+ describe: "Filter telemetry events by status (success,error)",
+ type: "string",
+ })
+ .option("since", {
+ describe: "Only include telemetry events after this time (relative like 1d or ISO timestamp)",
+ type: "string",
+ })
+ .option("until", {
+ describe: "Only include telemetry events before this time",
+ type: "string",
+ })
+ .option("compare", {
+ describe: "Path to baseline JSON created with --json for comparison",
+ type: "string",
+ })
+ .option("warn-latency", {
+ describe: "Warn if any tool p95 latency exceeds this many milliseconds",
+ type: "number",
+ })
+ .option("warn-errors", {
+ describe: "Warn if total errors exceed this count",
+ type: "number",
+ }),
+ handler: async (args) => {
+ await bootstrap(process.cwd(), async () => {
+ if (args.clear) {
+ await ToolHistory.clear()
+ console.log("Cleared telemetry history.")
+ }
+ const history = await ToolHistory.read()
+ const toolUsage = Object.fromEntries(Object.entries(history.tools).map(([tool, data]) => [tool, data.runs]))
+ const telemetryFilter = args.telemetry?.trim()
+ const statuses = parseList(args.status, true)
+ const since = args.since ? parseTimeInput(args.since) : undefined
+ const until = args.until ? parseTimeInput(args.until) : undefined
+ const telemetryEvents = (() => {
+ const base = (() => {
+ if (!telemetryFilter) return history.events
+ if (telemetryFilter === "all") return history.events
+ return history.events.filter((event) => event.id === telemetryFilter)
+ })()
+ return base.filter((event) => {
+ if (statuses.length > 0 && !statuses.includes(event.status)) return false
+ if (since !== undefined && event.timestamp < since) return false
+ if (until !== undefined && event.timestamp > until) return false
+ return true
+ })
+ })()
+ const limit = Math.max(1, args.limit ?? 20)
+ const limitedTelemetry = telemetryEvents.slice(-limit)
+ const telemetrySummary = summarizeTelemetry(telemetryEvents)
+
+ const sessionMetrics = await aggregateSessions()
+ const stats: SessionStats = {
+ ...sessionMetrics,
+ toolUsage,
+ toolTelemetry: telemetrySummary.tools,
+ }
+
+ const comparison = args.compare ? await compareBaseline(args.compare, telemetrySummary) : undefined
+ const warnings = collectWarnings(telemetrySummary, args.warnLatency, args.warnErrors)
+
+ if (args.json) {
+ const json = {
+ stats,
+ telemetry: limitedTelemetry,
+ telemetrySummary,
+ comparison,
+ warnings,
+ }
+ console.log(JSON.stringify(json, null, 2))
+ return
+ }
+
+ displayStats(stats)
+ displayTelemetryWindow(telemetrySummary)
+ if (telemetryFilter || telemetryEvents.length > 0) displayTelemetryEvents(limitedTelemetry)
+ if (args.details) {
+ displayTelemetryDetails(limitedTelemetry, {
+ format: args.detailsFormat ?? "pretty",
+ fields: parseList(args.fields),
+ })
+ }
+ if (comparison) displayComparison(comparison)
+ if (warnings.length > 0) {
+ for (const note of warnings) console.log(note)
+ const currentExit = typeof process.exitCode === "number" ? process.exitCode : 0
+ if (currentExit < 2) process.exitCode = 2
+ }
+ })
+ },
})
+async function aggregateSessions(): Promise> {
+ const sessions: Session.Info[] = []
+ for await (const info of Session.list()) {
+ sessions.push(info)
+ }
+
+ let totalMessages = 0
+ let totalCost = 0
+ let inputTokens = 0
+ let outputTokens = 0
+ let reasoningTokens = 0
+ let cacheReadTokens = 0
+ let cacheWriteTokens = 0
+
+ let earliest = sessions.length > 0 ? Math.min(...sessions.map((s) => s.time.created)) : Date.now()
+ let latest = sessions.length > 0 ? Math.max(...sessions.map((s) => s.time.updated)) : earliest
+
+ for (const session of sessions) {
+ earliest = Math.min(earliest, session.time.created)
+ latest = Math.max(latest, session.time.updated)
+ const messages = await Session.messages(session.id)
+ totalMessages += messages.length
+ for (const message of messages) {
+ if (message.info.role !== "assistant") continue
+ const assistant = message.info as MessageV2.Assistant
+ totalCost += assistant.cost ?? 0
+ inputTokens += assistant.tokens?.input ?? 0
+ outputTokens += assistant.tokens?.output ?? 0
+ reasoningTokens += assistant.tokens?.reasoning ?? 0
+ cacheReadTokens += assistant.tokens?.cache?.read ?? 0
+ cacheWriteTokens += assistant.tokens?.cache?.write ?? 0
+ }
+ }
+
+ const totalSessions = sessions.length
+ const dayMillis = 1000 * 60 * 60 * 24
+ const days = totalSessions > 0 ? Math.max(1, Math.ceil((latest - earliest) / dayMillis)) : 1
+ const costPerDay = days > 0 ? totalCost / days : 0
+
+ return {
+ totalSessions,
+ totalMessages,
+ totalCost,
+ totalTokens: {
+ input: inputTokens,
+ output: outputTokens,
+ reasoning: reasoningTokens,
+ cache: {
+ read: cacheReadTokens,
+ write: cacheWriteTokens,
+ },
+ },
+ dateRange: { earliest, latest },
+ days,
+ costPerDay,
+ }
+}
+
export function displayStats(stats: SessionStats) {
const width = 56
@@ -87,12 +336,450 @@ export function displayStats(stats: SessionStats) {
console.log("└────────────────────────────────────────────────────────┘")
}
console.log()
+
+ if (Object.keys(stats.toolTelemetry ?? {}).length === 0) return
+
+ console.log("┌─────────────────────── TOOL TELEMETRY ─────────────────────┐")
+ console.log("│ Tool Runs Avg P95 P99 Err% Success │")
+ console.log("├───────────────────────────────────────────────────────────┤")
+ const sorted = Object.entries(stats.toolTelemetry).sort(([, a], [, b]) => b.runs - a.runs)
+ for (const [tool, data] of sorted) {
+ const avg = formatDurationShort(data.averageDuration)
+ const p95 = formatDurationShort(data.p95Duration)
+ const p99 = formatDurationShort(data.p99Duration)
+ const errPercent = formatPercent(data.errorRate)
+ const successPercent = formatPercent(data.successRate)
+ const line = `│ ${tool.padEnd(10)} ${String(data.runs).padStart(4)} ${avg.padEnd(7)} ${p95.padEnd(7)} ${p99.padEnd(7)} ${errPercent.padEnd(
+ 6,
+ )} ${successPercent.padEnd(7)} │`
+ console.log(line)
+ }
+ console.log("└───────────────────────────────────────────────────────────┘")
+ console.log()
}
+
function formatNumber(num: number): string {
- if (num >= 1000000) {
- return (num / 1000000).toFixed(1) + "M"
- } else if (num >= 1000) {
- return (num / 1000).toFixed(1) + "K"
- }
+ if (num >= 1000000) return (num / 1000000).toFixed(1) + "M"
+ if (num >= 1000) return (num / 1000).toFixed(1) + "K"
return num.toString()
}
+
+function parseList(value?: string, lowercase = false): string[] {
+ if (!value) return []
+ return value
+ .split(/[\s,]+/g)
+ .map((item) => item.trim())
+ .filter(Boolean)
+ .map((item) => (lowercase ? item.toLowerCase() : item))
+}
+
+function parseTimeInput(value: string): number | undefined {
+ const text = value.trim()
+ if (!text) return undefined
+ if (text === "now") return Date.now()
+ const rel = text.match(/^(\d+)([smhdw])$/i)
+ if (rel) {
+ const amount = Number(rel[1])
+ const unit = rel[2].toLowerCase()
+ const factor = (() => {
+ if (unit === "s") return 1000
+ if (unit === "m") return 1000 * 60
+ if (unit === "h") return 1000 * 60 * 60
+ if (unit === "d") return 1000 * 60 * 60 * 24
+ if (unit === "w") return 1000 * 60 * 60 * 24 * 7
+ return 0
+ })()
+ return Date.now() - amount * factor
+ }
+ const date = new Date(text)
+ if (!Number.isNaN(date.getTime())) return date.getTime()
+ const numeric = Number(text)
+ if (!Number.isNaN(numeric)) return numeric
+ return undefined
+}
+
+function summarizeTelemetry(events: TelemetryEvent[]): TelemetrySummary {
+ if (events.length === 0) return { totalRuns: 0, totalErrors: 0, tools: {} }
+
+ const byTool = new Map()
+ for (const event of events) {
+ const existing = byTool.get(event.id)
+ const entry = existing ?? { runs: 0, errors: 0, totalDuration: 0, durations: [] as number[] }
+ if (!existing) byTool.set(event.id, entry)
+ entry.runs += 1
+ entry.totalDuration += event.duration
+ entry.durations.push(event.duration)
+ if (event.status === "error") entry.errors += 1
+ }
+
+ const timestamps = events.map((event) => event.timestamp)
+ const windowStart = Math.min(...timestamps)
+ const windowEnd = Math.max(...timestamps)
+ const totalErrors = events.filter((event) => event.status === "error").length
+ const tools: Record = {}
+
+ for (const [tool, entry] of byTool.entries()) {
+ const durations = entry.durations.toSorted((a, b) => a - b)
+ const runs = entry.runs
+ const errors = entry.errors
+ const avg = runs > 0 ? entry.totalDuration / runs : 0
+ const median = percentileFromSorted(durations, 50)
+ const p95 = percentileFromSorted(durations, 95)
+ const p99 = percentileFromSorted(durations, 99)
+ const errorRate = runs > 0 ? errors / runs : 0
+ const successRate = 1 - errorRate
+ tools[tool] = {
+ runs,
+ errors,
+ totalDuration: entry.totalDuration,
+ averageDuration: avg,
+ medianDuration: median,
+ p95Duration: p95,
+ p99Duration: p99,
+ errorRate,
+ successRate,
+ }
+ }
+
+ const rangeMs = windowEnd - windowStart
+ const perDayErrorRate = rangeMs > 0 ? totalErrors / Math.max(1, rangeMs / (1000 * 60 * 60 * 24)) : undefined
+
+ return {
+ windowStart,
+ windowEnd,
+ totalRuns: events.length,
+ totalErrors,
+ perDayErrorRate,
+ tools,
+ }
+}
+
+async function compareBaseline(path: string, current: TelemetrySummary): Promise {
+ const file = Bun.file(path)
+ const exists = await file.exists()
+ if (!exists) return undefined
+ const text = await file.text()
+ const payload = JSON.parse(text)
+ const baselineCandidate = (() => {
+ if (payload.telemetrySummary) return payload.telemetrySummary
+ if (Array.isArray(payload.telemetry)) return summarizeTelemetry(payload.telemetry as TelemetryEvent[])
+ if (Array.isArray(payload)) return summarizeTelemetry(payload as TelemetryEvent[])
+ return undefined
+ })()
+ if (!baselineCandidate) return undefined
+ const baseline = normalizeTelemetrySummary(baselineCandidate)
+ if (!baseline) return undefined
+ return makeComparison(path, baseline, current)
+}
+
+function makeComparison(path: string, baseline: TelemetrySummary, current: TelemetrySummary): TelemetryComparison {
+ const tools = new Set([...Object.keys(baseline.tools), ...Object.keys(current.tools)])
+ const toolComparisons = Array.from(tools)
+ .map((tool) => ({
+ tool,
+ baseline: baseline.tools[tool],
+ current: current.tools[tool],
+ }))
+ .filter((entry) => entry.baseline || entry.current)
+
+ return {
+ path,
+ totalRunsDelta: current.totalRuns - baseline.totalRuns,
+ totalErrorsDelta: current.totalErrors - baseline.totalErrors,
+ toolComparisons,
+ }
+}
+
+function collectWarnings(summary: TelemetrySummary, warnLatency?: number, warnErrors?: number): string[] {
+ const notes: string[] = []
+ if (warnLatency !== undefined) {
+ const offenders = Object.entries(summary.tools).filter(([, data]) => data.p95Duration > warnLatency)
+ for (const [tool, data] of offenders) {
+ notes.push(`⚠ ${tool} p95 ${formatDurationShort(data.p95Duration)} exceeds ${formatDurationShort(warnLatency)}`)
+ }
+ }
+ if (warnErrors !== undefined && summary.totalErrors > warnErrors) {
+ notes.push(`⚠ Total telemetry errors ${summary.totalErrors} exceed ${warnErrors}`)
+ }
+ return notes
+}
+
+function displayTelemetryWindow(summary: TelemetrySummary) {
+ if (summary.totalRuns === 0) {
+ console.log("No telemetry events recorded for the selected window.")
+ console.log()
+ return
+ }
+ const start = summary.windowStart ? formatTimestamp(summary.windowStart) : "unknown"
+ const end = summary.windowEnd ? formatTimestamp(summary.windowEnd) : "unknown"
+ const windowLine = `Telemetry window: ${start} → ${end}`
+ console.log(windowLine)
+ const metrics = [`runs ${summary.totalRuns}`, `errors ${summary.totalErrors}`]
+ if (summary.perDayErrorRate !== undefined) metrics.push(`errors/day ${summary.perDayErrorRate.toFixed(2)}`)
+ console.log(metrics.join(" • "))
+ console.log()
+}
+
+function displayComparison(comparison: TelemetryComparison) {
+ console.log(`Baseline comparison (${comparison.path}):`)
+ console.log(
+ [
+ ` total runs ${formatSigned(comparison.totalRunsDelta)}`,
+ ` total errors ${formatSigned(comparison.totalErrorsDelta)}`,
+ ].join(" • "),
+ )
+ if (comparison.toolComparisons.length === 0) {
+ console.log()
+ return
+ }
+ for (const item of comparison.toolComparisons.sort((a, b) => a.tool.localeCompare(b.tool))) {
+ const current = item.current
+ const baseline = item.baseline
+ if (!current && !baseline) continue
+ const runsDelta = current && baseline ? current.runs - baseline.runs : current ? current.runs : -baseline!.runs
+ const p95Delta = (() => {
+ if (current && baseline) return current.p95Duration - baseline.p95Duration
+ if (current) return current.p95Duration
+ return -baseline!.p95Duration
+ })()
+ const errorRateDelta = (() => {
+ if (current && baseline) return current.errorRate - baseline.errorRate
+ if (current) return current.errorRate
+ return -baseline!.errorRate
+ })()
+ const parts = [`${item.tool}: Δruns ${formatSigned(runsDelta)}`]
+ parts.push(`Δp95 ${formatSignedDuration(p95Delta)}`)
+ parts.push(`Δerr ${formatSignedPercent(errorRateDelta)}`)
+ if (current && baseline && baseline.p95Duration > 0) {
+ const ratio = current.p95Duration / baseline.p95Duration
+ if (ratio >= 3) parts.push(`⚠ p95 ${ratio.toFixed(1)}x`)
+ }
+ if (current && baseline && baseline.errorRate > 0) {
+ const ratio = current.errorRate / baseline.errorRate
+ if (ratio >= 3) parts.push(`⚠ err ${ratio.toFixed(1)}x`)
+ }
+ if (current && !baseline && current.runs > 0) parts.push("⚠ new tool")
+ if (!current && baseline && baseline.runs > 0) parts.push("⚠ missing tool")
+ console.log(parts.join(" • "))
+ }
+ console.log()
+}
+
+function displayTelemetryEvents(events: TelemetryEvent[]) {
+ if (events.length === 0) {
+ console.log("No telemetry events match the provided filter.")
+ return
+ }
+ console.log("┌──────────────────────── TELEMETRY EVENTS ─────────────────────────┐")
+ console.log("│ Time Tool Status Duration Session Message │")
+ console.log("├──────────────────────────────────────────────────────────────────┤")
+ for (const event of events) {
+ const date = formatTimestamp(event.timestamp)
+ const status = event.status === "success" ? "OK" : "ERR"
+ const duration = formatDurationShort(event.duration)
+ const session = event.sessionID.slice(-8)
+ const message = event.error ? event.error.slice(0, 30) : ""
+ const line = `│ ${date} ${event.id.padEnd(6)} ${status.padEnd(6)} ${duration.padEnd(8)} ${session.padEnd(8)} ${message.padEnd(
+ 22,
+ )} │`
+ console.log(line)
+ }
+ console.log("└──────────────────────────────────────────────────────────────────┘")
+}
+
+function displayTelemetryDetails(events: TelemetryEvent[], options: DetailOptions) {
+ if (events.length === 0) {
+ console.log("No telemetry metadata found for the selected events.")
+ return
+ }
+
+ if (options.format === "ndjson") {
+ for (const event of events) {
+ const extra = filterExtra(event.extra, options.fields)
+ const payload = {
+ timestamp: event.timestamp,
+ time: formatTimestamp(event.timestamp),
+ tool: event.id,
+ status: event.status,
+ duration: event.duration,
+ session: event.sessionID,
+ call: event.callID,
+ error: event.error,
+ extra,
+ }
+ console.log(JSON.stringify(payload))
+ }
+ return
+ }
+
+ if (options.format === "csv") {
+ const baseFields = ["timestamp", "time", "tool", "status", "duration", "session", "call", "error"]
+ const extraKeys = collectFieldNames(events, options.fields)
+ const header = [...baseFields, ...extraKeys]
+ console.log(header.join(","))
+ for (const event of events) {
+ const baseRow = [
+ String(event.timestamp),
+ formatTimestamp(event.timestamp),
+ event.id,
+ event.status,
+ String(event.duration),
+ event.sessionID,
+ event.callID ?? "",
+ event.error ?? "",
+ ]
+ const extra = filterExtra(event.extra, options.fields)
+ const extras = extraKeys.map((key) => toCSVValue(extra?.[key]))
+ console.log([...baseRow, ...extras].join(","))
+ }
+ return
+ }
+
+ console.log("Telemetry details:")
+ const hasMetadata = events.some((event) => {
+ const extra = filterExtra(event.extra, options.fields)
+ if (!extra) return false
+ return Object.keys(extra).length > 0
+ })
+ if (!hasMetadata) {
+ console.log("No telemetry metadata found for the selected events.")
+ return
+ }
+ for (const event of events) {
+ const extra = filterExtra(event.extra, options.fields)
+ if (!extra || Object.keys(extra).length === 0) continue
+ const header = `${formatTimestamp(event.timestamp)} ${event.id} (${event.status})`
+ console.log(header)
+ console.log(` session: ${event.sessionID} # opencode run --session ${event.sessionID}`)
+ if (event.callID) console.log(` call: ${event.callID}`)
+ console.log(` duration: ${formatDurationShort(event.duration)}`)
+ if (event.error) console.log(` error: ${event.error}`)
+ for (const key of Object.keys(extra).sort()) {
+ console.log(` ${key}: ${formatValue(extra[key])}`)
+ }
+ console.log()
+ }
+}
+
+function formatValue(value: unknown): string {
+ if (value === null) return "null"
+ if (typeof value === "number" || typeof value === "boolean") return String(value)
+ if (typeof value === "string") return value
+ if (Array.isArray(value)) return value.map((item) => formatValue(item)).join(", ")
+ return JSON.stringify(value)
+}
+
+function filterExtra(extra: TelemetryEvent["extra"], fields: string[]) {
+ if (!extra) return undefined
+ if (fields.length === 0) return extra
+ const picked: Record = {}
+ for (const key of fields) {
+ if (key in extra) picked[key] = extra[key]
+ }
+ return picked
+}
+
+function collectFieldNames(events: TelemetryEvent[], requested: string[]): string[] {
+ if (requested.length > 0) return Array.from(new Set(requested))
+ const names = new Set()
+ for (const event of events) {
+ if (!event.extra) continue
+ Object.keys(event.extra).forEach((key) => names.add(key))
+ }
+ return Array.from(names).sort()
+}
+
+function toCSVValue(value: unknown): string {
+ if (value === undefined) return ""
+ const raw = formatValue(value)
+ if (raw.includes(",") || raw.includes('"')) return `"${raw.replace(/"/g, '""')}"`
+ return raw
+}
+
+function percentileFromSorted(values: number[], target: number): number {
+ if (values.length === 0) return 0
+ if (values.length === 1) return values[0]
+ const rank = (target / 100) * (values.length - 1)
+ const lower = Math.floor(rank)
+ const upper = Math.ceil(rank)
+ if (lower === upper) return values[lower]
+ const weight = rank - lower
+ return values[lower] * (1 - weight) + values[upper] * weight
+}
+
+function formatDurationShort(duration: number): string {
+ if (duration < 1000) return `${duration.toFixed(0)}ms`
+ if (duration < 60000) return `${(duration / 1000).toFixed(2)}s`
+ return `${(duration / 60000).toFixed(2)}m`
+}
+
+function formatSigned(value: number): string {
+ if (value === 0) return "±0"
+ return value > 0 ? `+${value}` : `${value}`
+}
+
+function formatSignedDuration(value: number): string {
+ if (value === 0) return "±0ms"
+ const label = formatDurationShort(Math.abs(value))
+ return value > 0 ? `+${label}` : `-${label}`
+}
+
+function formatPercent(value: number): string {
+ return `${(value * 100).toFixed(1)}%`
+}
+
+function formatSignedPercent(value: number): string {
+ if (value === 0) return "±0.0%"
+ const abs = (Math.abs(value) * 100).toFixed(1)
+ return value > 0 ? `+${abs}%` : `-${abs}%`
+}
+
+function formatTimestamp(timestamp: number): string {
+ return new Date(timestamp).toISOString().replace("T", " ").split(".")[0]
+}
+
+function normalizeTelemetrySummary(input: any): TelemetrySummary | undefined {
+ if (!input || typeof input !== "object") return undefined
+ const rawTools = (input as any).tools
+ const tools: Record = {}
+ if (rawTools && typeof rawTools === "object") {
+ for (const [tool, raw] of Object.entries(rawTools as Record)) {
+ const runs = Number(raw?.runs ?? 0)
+ const errors = Number(raw?.errors ?? 0)
+ const totalDuration = Number(raw?.totalDuration ?? 0)
+ const avg = Number(raw?.averageDuration ?? (runs > 0 ? totalDuration / runs : 0))
+ const median = Number(raw?.medianDuration ?? avg)
+ const p95 = Number(raw?.p95Duration ?? median)
+ const p99 = Number(raw?.p99Duration ?? p95)
+ const errorRate = runs > 0 ? errors / runs : 0
+ const successRate = 1 - errorRate
+ tools[tool] = {
+ runs,
+ errors,
+ totalDuration,
+ averageDuration: avg,
+ medianDuration: median,
+ p95Duration: p95,
+ p99Duration: p99,
+ errorRate,
+ successRate,
+ }
+ }
+ }
+ const totals = Object.values(tools)
+ const totalRuns =
+ typeof input.totalRuns === "number" ? input.totalRuns : totals.reduce((sum, entry) => sum + entry.runs, 0)
+ const totalErrors =
+ typeof input.totalErrors === "number" ? input.totalErrors : totals.reduce((sum, entry) => sum + entry.errors, 0)
+ const perDayErrorRate = typeof input.perDayErrorRate === "number" ? input.perDayErrorRate : undefined
+ const windowStart = typeof input.windowStart === "number" ? input.windowStart : undefined
+ const windowEnd = typeof input.windowEnd === "number" ? input.windowEnd : undefined
+ return {
+ windowStart,
+ windowEnd,
+ totalRuns,
+ totalErrors,
+ perDayErrorRate,
+ tools,
+ }
+}
diff --git a/packages/opencode/src/cli/ui.ts b/packages/opencode/src/cli/ui.ts
index bdbaed911b..caab8383bb 100644
--- a/packages/opencode/src/cli/ui.ts
+++ b/packages/opencode/src/cli/ui.ts
@@ -4,9 +4,9 @@ import { NamedError } from "../util/error"
export namespace UI {
const LOGO = [
- [`█▀▀█ █▀▀█ █▀▀ █▀▀▄ `, `█▀▀ █▀▀█ █▀▀▄ █▀▀`],
- [`█░░█ █░░█ █▀▀ █░░█ `, `█░░ █░░█ █░░█ █▀▀`],
- [`▀▀▀▀ █▀▀▀ ▀▀▀ ▀ ▀ `, `▀▀▀ ▀▀▀▀ ▀▀▀ ▀▀▀`],
+ [`█▀▀▀ █▀▀█ ░▀░ █▀▄▀█ █▀▀█ ░▀░ █▀▀█ █▀▀`],
+ [`█░▀█ █▄▄▀ ▀█▀ █░▀░█ █░░█ ▀█▀ █▄▄▀ █▀▀`],
+ [`▀▀▀▀ ▀░▀▀ ▀▀▀ ▀░░░▀ ▀▀▀▀ ▀▀▀ ▀░▀▀ ▀▀▀`],
]
export const CancelledError = NamedError.create("UICancelledError", z.void())
@@ -50,9 +50,8 @@ export namespace UI {
for (const row of LOGO) {
if (pad) result.push(pad)
result.push(Bun.color("gray", "ansi"))
- result.push(row[0])
+ result.push(row)
result.push("\x1b[0m")
- result.push(row[1])
result.push(EOL)
}
return result.join("").trimEnd()
diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts
index 40e4d90a4a..967d655890 100644
--- a/packages/opencode/src/config/config.ts
+++ b/packages/opencode/src/config/config.ts
@@ -131,7 +131,7 @@ export namespace Config {
if (!ALLOWED_DIRS.has(dirname)) {
throw new InvalidError({
path: dir,
- message: `Unexpected directory "${dirname}" found in "${dir}". Only ${ALLOWED_DIRS.values().toArray().join(", ")} directories are allowed.`,
+ message: `Unexpected directory "${dirname}" found in "${dir}". Only ${Array.from(ALLOWED_DIRS).join(", ")} directories are allowed.`,
})
}
}
@@ -302,6 +302,8 @@ export namespace Config {
edit: Permission.optional(),
bash: z.union([Permission, z.record(z.string(), Permission)]).optional(),
webfetch: Permission.optional(),
+ fetchurl: Permission.optional(),
+ websearch: Permission.optional(),
})
.optional(),
})
@@ -533,6 +535,8 @@ export namespace Config {
edit: Permission.optional(),
bash: z.union([Permission, z.record(z.string(), Permission)]).optional(),
webfetch: Permission.optional(),
+ fetchurl: Permission.optional(),
+ websearch: Permission.optional(),
})
.optional(),
tools: z.record(z.string(), z.boolean()).optional(),
diff --git a/packages/opencode/src/evaluation/baseline.ts b/packages/opencode/src/evaluation/baseline.ts
new file mode 100644
index 0000000000..0f92f14644
--- /dev/null
+++ b/packages/opencode/src/evaluation/baseline.ts
@@ -0,0 +1,593 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import { Bus } from "../bus"
+import type { Trace } from "../trace"
+import { EvaluationEngine } from "./engine"
+
+/**
+ * Baseline management for comparative analysis and regression detection.
+ *
+ * Baselines serve as reference points for tracking metric performance over time.
+ * They enable:
+ * - Regression detection by comparing new traces to established baselines
+ * - A/B testing by comparing two different configurations
+ * - Performance tracking across versions/iterations
+ * - Statistical analysis of metric distributions
+ *
+ * @example
+ * ```typescript
+ * // Create a baseline from current production performance
+ * const baseline = await Baseline.create({
+ * id: "prod-v1.0",
+ * name: "Production Baseline v1.0",
+ * description: "Performance baseline for initial release",
+ * metricIDs: ["error-rate", "response-time", "cost"],
+ * tags: ["production", "v1.0"]
+ * })
+ *
+ * // Add traces to the baseline
+ * await Baseline.addTrace(baseline.id, trace)
+ *
+ * // Compare new trace against baseline
+ * const comparison = await Baseline.compare(baseline.id, newTrace)
+ * if (comparison.regressions.length > 0) {
+ * console.warn("Performance regression detected!")
+ * }
+ * ```
+ */
+export namespace Baseline {
+ /**
+ * Statistical summary of metric values in a baseline.
+ */
+ export const Statistics = z.object({
+ metricID: z.string(),
+ count: z.number(),
+ mean: z.number(),
+ median: z.number(),
+ stdDev: z.number(),
+ min: z.number(),
+ max: z.number(),
+ p50: z.number(),
+ p95: z.number(),
+ p99: z.number(),
+ })
+ export type Statistics = z.infer
+
+ /**
+ * A baseline definition with reference trace data.
+ */
+ export const Definition = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string(),
+
+ // Metrics to track in this baseline
+ metricIDs: z.array(z.string()),
+
+ // Reference traces
+ traceIDs: z.array(z.string()).default([]),
+
+ // Computed statistics
+ statistics: z.array(Statistics).default([]),
+
+ // Configuration
+ minSampleSize: z.number().default(10),
+ regressionThreshold: z.number().default(0.1), // 10% degradation
+
+ // Metadata
+ tags: z.array(z.string()).default([]),
+ createdAt: z.number(),
+ updatedAt: z.number(),
+ version: z.string().default("1.0.0"),
+ })
+ export type Definition = z.infer
+
+ /**
+ * Result of comparing a trace against a baseline.
+ */
+ export const ComparisonResult = z.object({
+ baselineID: z.string(),
+ traceID: z.string(),
+
+ // Per-metric comparison
+ metrics: z.array(
+ z.object({
+ metricID: z.string(),
+ baselineValue: z.number(), // Mean from baseline
+ traceValue: z.number(),
+ delta: z.number(), // Absolute difference
+ percentChange: z.number(), // Percentage change
+ isRegression: z.boolean(),
+ zScore: z.number().optional(), // How many std devs from mean
+ }),
+ ),
+
+ // Summary
+ regressions: z.array(z.string()), // Metric IDs with regressions
+ improvements: z.array(z.string()), // Metric IDs with improvements
+ overallScore: z.number(), // 0-1, weighted average of metrics
+
+ timestamp: z.number(),
+ })
+ export type ComparisonResult = z.infer
+
+ /**
+ * A/B test comparison between two baselines.
+ */
+ export const ABTestResult = z.object({
+ baselineA: z.string(),
+ baselineB: z.string(),
+
+ // Per-metric statistical comparison
+ metrics: z.array(
+ z.object({
+ metricID: z.string(),
+ meanA: z.number(),
+ meanB: z.number(),
+ medianA: z.number(),
+ medianB: z.number(),
+ delta: z.number(),
+ percentChange: z.number(),
+ winner: z.enum(["A", "B", "tie"]),
+ confidence: z.number(), // 0-1, statistical confidence
+ }),
+ ),
+
+ // Overall winner
+ overallWinner: z.enum(["A", "B", "tie"]),
+ sampleSizeA: z.number(),
+ sampleSizeB: z.number(),
+
+ timestamp: z.number(),
+ })
+ export type ABTestResult = z.infer
+
+ export const Event = {
+ Created: Bus.event(
+ "baseline.created",
+ z.object({
+ baselineID: z.string(),
+ }),
+ ),
+ Updated: Bus.event(
+ "baseline.updated",
+ z.object({
+ baselineID: z.string(),
+ }),
+ ),
+ RegressionDetected: Bus.event(
+ "baseline.regression",
+ z.object({
+ baselineID: z.string(),
+ traceID: z.string(),
+ regressions: z.array(z.string()),
+ }),
+ ),
+ }
+
+ /**
+ * Create a new baseline.
+ *
+ * @param baseline - The baseline configuration
+ * @returns The created baseline definition
+ *
+ * @example
+ * ```typescript
+ * const baseline = await Baseline.create({
+ * id: "prod-baseline",
+ * name: "Production Baseline",
+ * description: "Reference performance for production",
+ * metricIDs: ["error-rate", "latency"],
+ * tags: ["production"]
+ * })
+ * ```
+ */
+ export async function create(
+ baseline: Pick &
+ Partial>
+ ): Promise {
+ const now = Date.now()
+ const complete: Definition = {
+ traceIDs: [],
+ statistics: [],
+ minSampleSize: 10,
+ regressionThreshold: 0.1,
+ tags: [],
+ version: "1.0.0",
+ ...baseline,
+ createdAt: now,
+ updatedAt: now,
+ }
+
+ await Storage.write(["baseline", baseline.id], complete)
+ Bus.publish(Event.Created, { baselineID: baseline.id })
+
+ return complete
+ }
+
+ /**
+ * Get a baseline by ID.
+ *
+ * @param id - The baseline ID
+ * @returns The baseline definition
+ */
+ export async function get(id: string): Promise {
+ return Storage.read(["baseline", id])
+ }
+
+ /**
+ * List all baselines.
+ *
+ * @returns Array of baseline definitions
+ */
+ export async function list(): Promise {
+ const keys = await Storage.list(["baseline"])
+ const baselines: Definition[] = []
+
+ for (const key of keys) {
+ const baseline = await Storage.read(key)
+ baselines.push(baseline)
+ }
+
+ return baselines.sort((a, b) => b.updatedAt - a.updatedAt)
+ }
+
+ /**
+ * Add a trace to a baseline and update statistics.
+ *
+ * Evaluates the trace against all baseline metrics and updates
+ * the statistical distribution.
+ *
+ * @param baselineID - The baseline ID
+ * @param trace - The trace to add
+ *
+ * @example
+ * ```typescript
+ * await Baseline.addTrace("prod-baseline", trace)
+ * ```
+ */
+ export async function addTrace(baselineID: string, trace: Trace.Complete): Promise {
+ const baseline = await get(baselineID)
+ const { Metric } = await import("./metric")
+
+ // Get all metrics for this baseline
+ const metrics = await Promise.all(baseline.metricIDs.map((id) => Metric.get(id)))
+
+ // Evaluate trace against all metrics
+ await EvaluationEngine.evaluateMany(trace, metrics)
+
+ // Add trace to baseline
+ baseline.traceIDs.push(trace.id)
+
+ // Update statistics
+ baseline.statistics = await computeStatistics(baselineID, baseline.metricIDs)
+ baseline.updatedAt = Date.now()
+
+ await Storage.write(["baseline", baselineID], baseline)
+ Bus.publish(Event.Updated, { baselineID })
+ }
+
+ /**
+ * Add multiple traces to a baseline in a single batch operation.
+ * Much faster than calling addTrace() in a loop.
+ *
+ * @param baselineID - The baseline ID
+ * @param traces - Array of traces to add
+ *
+ * @example
+ * ```typescript
+ * const historicalTraces = loadHistoricalData()
+ * await Baseline.addTraces("prod-baseline", historicalTraces)
+ * ```
+ */
+ export async function addTraces(
+ baselineID: string,
+ traces: Trace.Complete[]
+ ): Promise {
+ const baseline = await get(baselineID)
+ const { Metric } = await import("./metric")
+
+ // Get all metrics for this baseline
+ const metrics = await Promise.all(baseline.metricIDs.map((id) => Metric.get(id)))
+
+ // Evaluate all traces in parallel
+ await Promise.all(
+ traces.map(async (trace) => {
+ await EvaluationEngine.evaluateMany(trace, metrics)
+ })
+ )
+
+ // Add all trace IDs
+ baseline.traceIDs.push(...traces.map((t) => t.id))
+
+ // Update statistics once for all new data
+ baseline.statistics = await computeStatistics(baselineID, baseline.metricIDs)
+ baseline.updatedAt = Date.now()
+
+ await Storage.write(["baseline", baselineID], baseline)
+ Bus.publish(Event.Updated, { baselineID })
+ }
+
+ /**
+ * Compare a trace against a baseline.
+ *
+ * Evaluates the trace and compares each metric against the baseline's
+ * statistical distribution to detect regressions or improvements.
+ *
+ * @param baselineID - The baseline to compare against
+ * @param trace - The trace to evaluate
+ * @returns Comparison result with regression detection
+ *
+ * @example
+ * ```typescript
+ * const comparison = await Baseline.compare("prod-baseline", trace)
+ * if (comparison.regressions.length > 0) {
+ * console.error(`Regressions detected: ${comparison.regressions.join(", ")}`)
+ * }
+ * ```
+ */
+ export async function compare(baselineID: string, trace: Trace.Complete): Promise {
+ const baseline = await get(baselineID)
+ const { Metric } = await import("./metric")
+
+ if (baseline.traceIDs.length < baseline.minSampleSize) {
+ throw new Error(`Baseline ${baselineID} needs at least ${baseline.minSampleSize} traces`)
+ }
+
+ // Get all metrics and evaluate trace
+ const metrics = await Promise.all(baseline.metricIDs.map((id) => Metric.get(id)))
+ const results = await EvaluationEngine.evaluateMany(trace, metrics)
+
+ const metricComparisons = []
+ const regressions: string[] = []
+ const improvements: string[] = []
+
+ for (const result of results) {
+ const stats = baseline.statistics.find((s) => s.metricID === result.metricID)
+ if (!stats) continue
+
+ const metric = metrics.find((m) => m.id === result.metricID)!
+ const traceValue = result.score
+ const baselineValue = stats.mean
+ const delta = traceValue - baselineValue
+ const percentChange = baselineValue === 0 ? 0 : (delta / baselineValue) * 100
+ const zScore = stats.stdDev === 0 ? 0 : delta / stats.stdDev
+
+ // Determine if this is a regression based on metric direction
+ const isWorse = metric.higherIsBetter ? delta < 0 : delta > 0
+
+ // For regression detection:
+ // - Use percent change if baseline is non-zero
+ // - Use absolute delta if baseline is zero (any change from 0 is significant)
+ const isRegression = baselineValue === 0
+ ? isWorse && Math.abs(delta) > baseline.regressionThreshold
+ : isWorse && Math.abs(percentChange) > baseline.regressionThreshold * 100
+
+ if (isRegression) {
+ regressions.push(result.metricID)
+ } else {
+ // Check for improvements using same logic
+ const isImprovement = baselineValue === 0
+ ? !isWorse && Math.abs(delta) > baseline.regressionThreshold
+ : !isWorse && Math.abs(percentChange) > baseline.regressionThreshold * 100
+
+ if (isImprovement) {
+ improvements.push(result.metricID)
+ }
+ }
+
+ metricComparisons.push({
+ metricID: result.metricID,
+ baselineValue,
+ traceValue,
+ delta,
+ percentChange,
+ isRegression,
+ zScore,
+ })
+ }
+
+ // Compute overall score (weighted average of normalized scores)
+ const overallScore = metricComparisons.reduce((sum, m) => {
+ const normalizedScore = m.isRegression ? 0 : 1
+ return sum + normalizedScore
+ }, 0) / metricComparisons.length
+
+ const comparisonResult: ComparisonResult = {
+ baselineID,
+ traceID: trace.id,
+ metrics: metricComparisons,
+ regressions,
+ improvements,
+ overallScore,
+ timestamp: Date.now(),
+ }
+
+ // Store comparison result
+ await Storage.write(["baseline-comparison", baselineID, trace.id], comparisonResult)
+
+ // Emit event if regressions detected
+ if (regressions.length > 0) {
+ Bus.publish(Event.RegressionDetected, {
+ baselineID,
+ traceID: trace.id,
+ regressions,
+ })
+ }
+
+ return comparisonResult
+ }
+
+ /**
+ * Compare two baselines for A/B testing.
+ *
+ * Performs statistical comparison between two baselines to determine
+ * which performs better across tracked metrics.
+ *
+ * @param baselineAID - First baseline ID
+ * @param baselineBID - Second baseline ID
+ * @returns A/B test comparison result
+ *
+ * @example
+ * ```typescript
+ * const result = await Baseline.compareAB("v1-baseline", "v2-baseline")
+ * console.log(`Winner: ${result.overallWinner}`)
+ * result.metrics.forEach(m => {
+ * console.log(`${m.metricID}: ${m.winner} wins by ${m.percentChange.toFixed(1)}%`)
+ * })
+ * ```
+ */
+ export async function compareAB(baselineAID: string, baselineBID: string): Promise {
+ const baselineA = await get(baselineAID)
+ const baselineB = await get(baselineBID)
+
+ if (baselineA.traceIDs.length < baselineA.minSampleSize) {
+ throw new Error(`Baseline A needs at least ${baselineA.minSampleSize} traces`)
+ }
+ if (baselineB.traceIDs.length < baselineB.minSampleSize) {
+ throw new Error(`Baseline B needs at least ${baselineB.minSampleSize} traces`)
+ }
+
+ const metricComparisons = []
+ let aWins = 0
+ let bWins = 0
+
+ // Compare each metric that exists in both baselines
+ const commonMetrics = baselineA.metricIDs.filter((id) => baselineB.metricIDs.includes(id))
+ const { Metric } = await import("./metric")
+
+ for (const metricID of commonMetrics) {
+ const statsA = baselineA.statistics.find((s) => s.metricID === metricID)
+ const statsB = baselineB.statistics.find((s) => s.metricID === metricID)
+
+ if (!statsA || !statsB) continue
+
+ const metric = await Metric.get(metricID)
+ const delta = statsB.mean - statsA.mean
+ const percentChange = statsA.mean === 0 ? 0 : (delta / statsA.mean) * 100
+
+ // Determine winner based on metric direction
+ let winner: "A" | "B" | "tie"
+ if (Math.abs(percentChange) < 1) {
+ winner = "tie"
+ } else if (metric.higherIsBetter) {
+ winner = delta > 0 ? "B" : "A"
+ } else {
+ winner = delta < 0 ? "B" : "A"
+ }
+
+ if (winner === "A") aWins++
+ if (winner === "B") bWins++
+
+ // Simple confidence based on sample size and effect size
+ const minSampleSize = Math.min(statsA.count, statsB.count)
+ const effectSize = Math.abs(delta) / Math.max(statsA.stdDev, statsB.stdDev, 1)
+ const confidence = Math.min(0.99, (minSampleSize / 100) * effectSize)
+
+ metricComparisons.push({
+ metricID,
+ meanA: statsA.mean,
+ meanB: statsB.mean,
+ medianA: statsA.median,
+ medianB: statsB.median,
+ delta,
+ percentChange,
+ winner,
+ confidence,
+ })
+ }
+
+ const overallWinner = aWins > bWins ? "A" : bWins > aWins ? "B" : "tie"
+
+ const result: ABTestResult = {
+ baselineA: baselineAID,
+ baselineB: baselineBID,
+ metrics: metricComparisons,
+ overallWinner,
+ sampleSizeA: baselineA.traceIDs.length,
+ sampleSizeB: baselineB.traceIDs.length,
+ timestamp: Date.now(),
+ }
+
+ // Store A/B test result
+ await Storage.write(["ab-test", `${baselineAID}-vs-${baselineBID}`, Date.now().toString()], result)
+
+ return result
+ }
+
+ /**
+ * Find baselines by tag.
+ *
+ * @param tag - The tag to filter by
+ * @returns Array of baselines with the specified tag
+ */
+ export async function findByTag(tag: string): Promise {
+ const all = await list()
+ return all.filter((b) => b.tags.includes(tag))
+ }
+
+ /**
+ * Remove a baseline.
+ *
+ * @param id - The baseline ID to remove
+ */
+ export async function remove(id: string): Promise {
+ await Storage.remove(["baseline", id])
+ }
+
+ /**
+ * Compute statistics for a baseline's metrics.
+ *
+ * @param baselineID - The baseline ID
+ * @param metricIDs - Metric IDs to compute statistics for
+ * @returns Array of statistics per metric
+ */
+ async function computeStatistics(baselineID: string, metricIDs: string[]): Promise {
+ const stats: Statistics[] = []
+ const baseline = await get(baselineID)
+ const metricSet = new Set(metricIDs)
+ const scoresByMetric = new Map()
+
+ for (const traceID of baseline.traceIDs) {
+ const results = await EvaluationEngine.getResults(traceID)
+ for (const result of results) {
+ if (!metricSet.has(result.metricID)) continue
+ const existing = scoresByMetric.get(result.metricID) ?? []
+ existing.push(result.score)
+ scoresByMetric.set(result.metricID, existing)
+ }
+ }
+
+ for (const metricID of metricIDs) {
+ const scores = scoresByMetric.get(metricID)
+ if (!scores || scores.length === 0) continue
+ const sorted = [...scores].sort((a, b) => a - b)
+ const count = sorted.length
+
+ const mean = sorted.reduce((sum, s) => sum + s, 0) / count
+ const median = sorted[Math.floor(count / 2)]
+ const variance = sorted.reduce((sum, s) => sum + Math.pow(s - mean, 2), 0) / count
+ const stdDev = Math.sqrt(variance)
+ const min = sorted[0]
+ const max = sorted[count - 1]
+ const p50 = sorted[Math.floor((count - 1) * 0.5)]
+ const p95 = sorted[Math.floor((count - 1) * 0.95)]
+ const p99 = sorted[Math.floor((count - 1) * 0.99)]
+
+ stats.push({
+ metricID,
+ count,
+ mean,
+ median,
+ stdDev,
+ min,
+ max,
+ p50,
+ p95,
+ p99,
+ })
+ }
+
+ return stats
+ }
+}
diff --git a/packages/opencode/src/evaluation/dataset.ts b/packages/opencode/src/evaluation/dataset.ts
new file mode 100644
index 0000000000..4bba091bce
--- /dev/null
+++ b/packages/opencode/src/evaluation/dataset.ts
@@ -0,0 +1,278 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import { Bus } from "../bus"
+
+/**
+ * Dataset management for test-driven evaluation.
+ *
+ * Datasets contain test cases with input scenarios and expected behaviors
+ * defined through assertions. Test cases can be:
+ * - Executed against traces to verify behavior
+ * - Tagged and filtered for organization
+ * - Enabled/disabled for selective testing
+ * - Versioned for tracking changes over time
+ *
+ * Supported assertion types:
+ * - tool-called: Verify specific tools were invoked
+ * - output-matches: Check output against regex patterns
+ * - output-contains: Verify substring presence
+ * - no-errors: Ensure error-free execution
+ * - duration-under: Performance threshold checks
+ * - cost-under: Budget constraints
+ * - metric-passes: Evaluate against registered metrics
+ * - custom: JavaScript expressions for flexible logic
+ *
+ * @example
+ * ```typescript
+ * await Dataset.create({
+ * id: "smoke-tests",
+ * name: "Smoke Test Suite",
+ * description: "Critical path validation",
+ * version: "1.0.0",
+ * testCases: [{
+ * id: "test-1",
+ * name: "Basic Task",
+ * input: { prompt: "List files", context: {} },
+ * assertions: [
+ * { type: "tool-called", toolID: "LS", minCount: 1 },
+ * { type: "no-errors" }
+ * ],
+ * tags: ["critical"],
+ * enabled: true
+ * }],
+ * tags: ["production"]
+ * })
+ * ```
+ */
+export namespace Dataset {
+ /**
+ * Assertion types for test cases.
+ *
+ * Assertions define expected trace behaviors and are evaluated
+ * against completed traces to determine test pass/fail status.
+ */
+ export const Assertion = z.discriminatedUnion("type", [
+ z.object({
+ type: z.literal("tool-called"),
+ toolID: z.string(),
+ minCount: z.number().optional(),
+ maxCount: z.number().optional(),
+ }),
+ z.object({
+ type: z.literal("output-matches"),
+ pattern: z.string(), // Regex pattern
+ flags: z.string().optional(),
+ }),
+ z.object({
+ type: z.literal("output-contains"),
+ substring: z.string(),
+ }),
+ z.object({
+ type: z.literal("no-errors"),
+ }),
+ z.object({
+ type: z.literal("duration-under"),
+ milliseconds: z.number(),
+ }),
+ z.object({
+ type: z.literal("cost-under"),
+ dollars: z.number(),
+ }),
+ z.object({
+ type: z.literal("metric-passes"),
+ metricID: z.string(),
+ }),
+ z.object({
+ type: z.literal("custom"),
+ expression: z.string(), // JavaScript expression evaluated against trace
+ description: z.string(),
+ }),
+ ])
+ export type Assertion = z.infer
+
+ /**
+ * Test case with input and expected behavior
+ */
+ export const TestCase = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string().optional(),
+
+ // Input
+ input: z.object({
+ prompt: z.string(),
+ context: z.record(z.string(), z.any()).optional(),
+ }),
+
+ // Expected behavior
+ assertions: z.array(Assertion),
+
+ // Metadata
+ tags: z.array(z.string()).default([]),
+ enabled: z.boolean().default(true),
+ })
+ export type TestCase = z.infer
+
+ /**
+ * Dataset definition
+ */
+ export const Definition = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string(),
+ version: z.string(),
+
+ testCases: z.array(TestCase),
+
+ // Metadata
+ tags: z.array(z.string()).default([]),
+ createdAt: z.number(),
+ updatedAt: z.number(),
+ })
+ export type Definition = z.infer
+
+ export const Event = {
+ Created: Bus.event(
+ "dataset.created",
+ z.object({
+ datasetID: z.string(),
+ }),
+ ),
+ Updated: Bus.event(
+ "dataset.updated",
+ z.object({
+ datasetID: z.string(),
+ }),
+ ),
+ }
+
+ /**
+ * Create a new dataset
+ */
+ export async function create(dataset: Omit): Promise {
+ const now = Date.now()
+ const complete: Definition = {
+ ...dataset,
+ createdAt: now,
+ updatedAt: now,
+ }
+
+ await Storage.write(["dataset", dataset.id], complete)
+ Bus.publish(Event.Created, { datasetID: dataset.id })
+ return complete
+ }
+
+ /**
+ * Update an existing dataset
+ */
+ export async function update(id: string, updates: Partial>): Promise {
+ const existing = await get(id)
+ const updated: Definition = {
+ ...existing,
+ ...updates,
+ updatedAt: Date.now(),
+ }
+
+ await Storage.write(["dataset", id], updated)
+ Bus.publish(Event.Updated, { datasetID: id })
+ return updated
+ }
+
+ /**
+ * Get a dataset by ID
+ */
+ export async function get(id: string): Promise {
+ const dataset = await Storage.read(["dataset", id])
+ return dataset
+ }
+
+ /**
+ * List all datasets
+ */
+ export async function list(): Promise {
+ const keys = await Storage.list(["dataset"])
+ const datasets: Definition[] = []
+
+ for (const key of keys) {
+ const dataset = await Storage.read(key)
+ datasets.push(dataset)
+ }
+
+ return datasets.sort((a, b) => b.updatedAt - a.updatedAt)
+ }
+
+ /**
+ * Check if a dataset exists
+ */
+ export async function exists(id: string): Promise {
+ try {
+ await get(id)
+ return true
+ } catch {
+ return false
+ }
+ }
+
+ /**
+ * Remove a dataset
+ */
+ export async function remove(id: string): Promise {
+ await Storage.remove(["dataset", id])
+ }
+
+ /**
+ * Find datasets by tag
+ */
+ export async function findByTag(tag: string): Promise {
+ const all = await list()
+ return all.filter((d) => d.tags.includes(tag))
+ }
+
+ /**
+ * Export dataset to JSON
+ */
+ export async function exportToJSON(id: string): Promise {
+ const dataset = await get(id)
+ return JSON.stringify(dataset, null, 2)
+ }
+
+ /**
+ * Import dataset from JSON
+ */
+ export async function importFromJSON(json: string): Promise {
+ const data = JSON.parse(json)
+ const dataset = Definition.parse(data)
+
+ // Check if exists and update, or create new
+ if (await exists(dataset.id)) {
+ return update(dataset.id, dataset)
+ }
+ return create(dataset)
+ }
+
+ /**
+ * Add a test case to a dataset
+ */
+ export async function addTestCase(datasetID: string, testCase: TestCase): Promise {
+ const dataset = await get(datasetID)
+ dataset.testCases.push(testCase)
+ return update(datasetID, { testCases: dataset.testCases })
+ }
+
+ /**
+ * Remove a test case from a dataset
+ */
+ export async function removeTestCase(datasetID: string, testCaseID: string): Promise {
+ const dataset = await get(datasetID)
+ dataset.testCases = dataset.testCases.filter((tc) => tc.id !== testCaseID)
+ return update(datasetID, { testCases: dataset.testCases })
+ }
+
+ /**
+ * Get enabled test cases from a dataset
+ */
+ export async function getEnabledTestCases(datasetID: string): Promise {
+ const dataset = await get(datasetID)
+ return dataset.testCases.filter((tc) => tc.enabled)
+ }
+}
diff --git a/packages/opencode/src/evaluation/engine.ts b/packages/opencode/src/evaluation/engine.ts
new file mode 100644
index 0000000000..16095b3e0b
--- /dev/null
+++ b/packages/opencode/src/evaluation/engine.ts
@@ -0,0 +1,284 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import { Bus } from "../bus"
+import type { Trace } from "../trace"
+import type { Metric } from "./metric"
+import { Heuristics } from "./heuristics"
+import { Log } from "../util/log"
+
+/**
+ * EvaluationEngine executes metric evaluations against traces.
+ *
+ * Supports three types of evaluators:
+ * - Rule: JavaScript expressions evaluated against trace data
+ * - Heuristic: Built-in functions for common metrics
+ * - LLM: AI-based evaluation using language models (planned)
+ *
+ * @example
+ * ```typescript
+ * const metric = await Metric.get("error-rate")
+ * const result = await EvaluationEngine.evaluate(trace, metric)
+ * console.log(`Score: ${result.score}, Passed: ${result.passed}`)
+ * ```
+ */
+export namespace EvaluationEngine {
+ const log = Log.create({ service: "evaluation-engine" })
+
+ export const Result = z.object({
+ id: z.string(),
+ traceID: z.string(),
+ metricID: z.string(),
+
+ score: z.number(),
+ passed: z.boolean(),
+
+ evaluatorType: z.enum(["rule", "heuristic", "llm"]),
+ reasoning: z.string().optional(),
+ metadata: z.record(z.string(), z.any()).optional(),
+
+ timestamp: z.number(),
+ })
+ export type Result = z.infer
+
+ export const Event = {
+ Completed: Bus.event(
+ "evaluation.completed",
+ z.object({
+ result: Result,
+ }),
+ ),
+ }
+
+ /**
+ * Evaluate a trace against a specific metric.
+ *
+ * Computes a score based on the metric's evaluator type and determines
+ * whether the trace passes the defined threshold.
+ *
+ * @param trace - The completed trace to evaluate
+ * @param metric - The metric definition containing evaluation logic and thresholds
+ * @returns Evaluation result with score, pass/fail status, and metadata
+ *
+ * @example
+ * ```typescript
+ * const metric = await Metric.get("response-time")
+ * const result = await EvaluationEngine.evaluate(trace, metric)
+ * if (result.passed) {
+ * console.log(`Passed with score: ${result.score}`)
+ * }
+ * ```
+ */
+ export async function evaluate(trace: Trace.Complete, metric: Metric.Definition): Promise {
+ log.debug("evaluating trace", {
+ traceID: trace.id,
+ metricID: metric.id,
+ })
+
+ const score = await computeScore(trace, metric)
+ const threshold = metric.threshold?.pass
+
+ let passed = true
+ if (threshold !== undefined) {
+ passed = metric.higherIsBetter ? score >= threshold : score <= threshold
+ }
+
+ const result: Result = {
+ id: Date.now().toString() + "-" + Math.random().toString(36).substring(7),
+ traceID: trace.id,
+ metricID: metric.id,
+ score,
+ passed,
+ evaluatorType: metric.evaluator.type,
+ timestamp: Date.now(),
+ }
+
+ // Store the result
+ await Storage.write(["evaluation", trace.id, result.id], result)
+
+ // Emit event
+ Bus.publish(Event.Completed, { result })
+
+ log.debug("evaluation completed", {
+ traceID: trace.id,
+ metricID: metric.id,
+ score,
+ passed,
+ })
+
+ return result
+ }
+
+ /**
+ * Evaluate a trace against multiple metrics in parallel.
+ *
+ * Efficiently evaluates multiple metrics simultaneously and returns
+ * all results. Useful for quality gates and comprehensive assessments.
+ *
+ * @param trace - The completed trace to evaluate
+ * @param metrics - Array of metric definitions to evaluate
+ * @returns Array of evaluation results, one per metric
+ *
+ * @example
+ * ```typescript
+ * const metrics = await Metric.findByTag("production")
+ * const results = await EvaluationEngine.evaluateMany(trace, metrics)
+ * const allPassed = results.every(r => r.passed)
+ * ```
+ */
+ export async function evaluateMany(
+ trace: Trace.Complete,
+ metrics: Metric.Definition[],
+ ): Promise {
+ return Promise.all(metrics.map((m) => evaluate(trace, m)))
+ }
+
+ /**
+ * Get evaluation results for a trace
+ */
+ export async function getResults(traceID: string): Promise {
+ const keys = await Storage.list(["evaluation", traceID])
+ const results: Result[] = []
+
+ for (const key of keys) {
+ const result = await Storage.read(key)
+ results.push(result)
+ }
+
+ return results.sort((a, b) => a.timestamp - b.timestamp)
+ }
+
+ /**
+ * Get evaluation results for a specific metric across traces
+ */
+ export async function getResultsForMetric(metricID: string): Promise {
+ // This requires scanning all evaluation results
+ // In a real implementation, you might want an index
+ const allKeys = await Storage.list(["evaluation"])
+ const results: Result[] = []
+
+ for (const key of allKeys) {
+ const result = await Storage.read(key)
+ if (result.metricID === metricID) {
+ results.push(result)
+ }
+ }
+
+ return results.sort((a, b) => a.timestamp - b.timestamp)
+ }
+
+ /**
+ * Compute score for a trace using a metric
+ */
+ async function computeScore(trace: Trace.Complete, metric: Metric.Definition): Promise {
+ switch (metric.evaluator.type) {
+ case "rule":
+ return evaluateRule(trace, metric.evaluator.expression)
+ case "heuristic":
+ return evaluateHeuristic(trace, metric.evaluator)
+ case "llm":
+ return evaluateLLM(trace, metric.evaluator)
+ }
+ }
+
+ /**
+ * Evaluate using a JavaScript rule expression
+ */
+ function evaluateRule(trace: Trace.Complete, expression: string): number {
+ try {
+ // Create a safe evaluation context
+ const func = new Function("trace", `return ${expression}`)
+ const result = func(trace)
+ // Convert boolean to number (true -> 1, false -> 0)
+ if (typeof result === "boolean") return result ? 1 : 0
+ return typeof result === "number" ? result : 0
+ } catch (error) {
+ log.error("rule evaluation failed", {
+ expression,
+ error: error instanceof Error ? error.message : String(error),
+ })
+ return 0
+ }
+ }
+
+ /**
+ * Evaluate using a built-in heuristic function
+ */
+ function evaluateHeuristic(trace: Trace.Complete, evaluator: Metric.HeuristicEvaluator): number {
+ const functionName = evaluator.function as keyof typeof Heuristics
+ const heuristic = Heuristics[functionName]
+
+ if (!heuristic) {
+ log.error("heuristic not found", {
+ function: evaluator.function,
+ })
+ return 0
+ }
+
+ try {
+ return heuristic(trace, evaluator.params)
+ } catch (error) {
+ log.error("heuristic evaluation failed", {
+ function: evaluator.function,
+ error: error instanceof Error ? error.message : String(error),
+ })
+ return 0
+ }
+ }
+
+ /**
+ * Evaluate using an LLM judge
+ * TODO: Implement LLM-based evaluation
+ */
+ async function evaluateLLM(_trace: Trace.Complete, evaluator: Metric.LLMEvaluator): Promise {
+ log.warn("LLM evaluation not yet implemented", {
+ model: evaluator.model,
+ })
+
+ // Placeholder - would call LLM API here
+ // const response = await callLLM(evaluator.model, {
+ // prompt: formatPrompt(evaluator.prompt, trace),
+ // })
+ // const parseFunc = new Function("output", evaluator.parseScore)
+ // return parseFunc(response)
+
+ return 0
+ }
+
+ /**
+ * Get summary statistics for evaluation results.
+ *
+ * Aggregates all evaluation results for a trace and computes summary
+ * statistics including pass/fail counts and average score.
+ *
+ * @param traceID - The ID of the trace to summarize
+ * @returns Summary object with statistics and full results
+ *
+ * @example
+ * ```typescript
+ * const summary = await EvaluationEngine.summarize("trace-123")
+ * console.log(`${summary.passed}/${summary.total} metrics passed`)
+ * console.log(`Average score: ${summary.averageScore.toFixed(2)}`)
+ * ```
+ */
+ export async function summarize(traceID: string): Promise<{
+ total: number
+ passed: number
+ failed: number
+ averageScore: number
+ results: Result[]
+ }> {
+ const results = await getResults(traceID)
+ const passed = results.filter((r) => r.passed).length
+ const failed = results.length - passed
+ const averageScore =
+ results.length > 0 ? results.reduce((sum, r) => sum + r.score, 0) / results.length : 0
+
+ return {
+ total: results.length,
+ passed,
+ failed,
+ averageScore,
+ results,
+ }
+ }
+}
diff --git a/packages/opencode/src/evaluation/feedback-manager.ts b/packages/opencode/src/evaluation/feedback-manager.ts
new file mode 100644
index 0000000000..9681f317fb
--- /dev/null
+++ b/packages/opencode/src/evaluation/feedback-manager.ts
@@ -0,0 +1,281 @@
+/**
+ * Intelligent feedback request manager.
+ *
+ * Requests user feedback at optimal times to maximize response rate
+ * while minimizing interruption.
+ *
+ * Strategy:
+ * - Only request feedback when session is idle
+ * - Focus on expensive or unusual operations
+ * - Rate-limit to avoid fatigue (max 1 per hour per user)
+ * - Track response rates and adjust strategy
+ */
+
+import { Bus } from "../bus"
+import { Session } from "../session"
+import { SessionPrompt } from "../session/prompt"
+import { Trace } from "../trace"
+import { Storage } from "../storage/storage"
+import { Log } from "../util/log"
+import { Telemetry } from "./telemetry"
+
+const log = Log.create({ service: "evaluation.feedback-manager" })
+
+export namespace FeedbackManager {
+ export type Strategy = {
+ /** Only ask for traces costing more than this (dollars) */
+ minCostThreshold: number
+ /** Only ask for traces longer than this (ms) */
+ minDurationThreshold: number
+ /** Ask for random traces this % of the time */
+ randomSamplingRate: number
+ /** Maximum feedback requests per hour */
+ maxRequestsPerHour: number
+ /** Minimum time since last request (ms) */
+ minTimeSinceLastRequest: number
+ }
+
+ const defaultStrategy: Strategy = {
+ minCostThreshold: 0.05, // $0.05
+ minDurationThreshold: 10000, // 10 seconds
+ randomSamplingRate: 0.05, // 5%
+ maxRequestsPerHour: 1,
+ minTimeSinceLastRequest: 60 * 60 * 1000, // 1 hour
+ }
+
+ let strategy: Strategy = defaultStrategy
+ let enabled = false
+ let unsubscribe: (() => void) | null = null
+
+ // Track last request time per session
+ const lastRequestTime = new Map()
+
+ // Track requests made this hour
+ let requestsThisHour = 0
+ let hourResetTimer: Timer | null = null
+
+ /**
+ * Enable feedback requests with optional custom strategy.
+ */
+ export function enable(customStrategy?: Partial) {
+ if (enabled) {
+ log.warn("feedback manager already enabled")
+ return
+ }
+
+ strategy = { ...defaultStrategy, ...customStrategy }
+ enabled = true
+
+ // Subscribe to idle events
+ unsubscribe = Bus.subscribe(SessionPrompt.Event.Idle, async ({ properties }) => {
+ try {
+ await handleIdleSession(properties.sessionID)
+ } catch (error) {
+ log.error("failed to handle idle session", { sessionID: properties.sessionID, error })
+ }
+ })
+
+ // Reset hourly counter
+ hourResetTimer = setInterval(() => {
+ requestsThisHour = 0
+ log.debug("reset hourly request counter")
+ }, 60 * 60 * 1000)
+
+ log.info("feedback manager enabled", { strategy })
+ }
+
+ /**
+ * Disable feedback requests.
+ */
+ export function disable() {
+ if (!enabled) return
+
+ enabled = false
+ if (unsubscribe) {
+ unsubscribe()
+ unsubscribe = null
+ }
+ if (hourResetTimer) {
+ clearInterval(hourResetTimer)
+ hourResetTimer = null
+ }
+
+ log.info("feedback manager disabled")
+ }
+
+ /**
+ * Check if we should request feedback for a trace.
+ */
+ function shouldRequestFeedback(trace: Trace.Complete, sessionID: string): boolean {
+ // Check rate limits
+ if (requestsThisHour >= strategy.maxRequestsPerHour) {
+ log.debug("skipping feedback: hourly limit reached")
+ return false
+ }
+
+ const lastRequest = lastRequestTime.get(sessionID) || 0
+ if (Date.now() - lastRequest < strategy.minTimeSinceLastRequest) {
+ log.debug("skipping feedback: too soon since last request")
+ return false
+ }
+
+ // Check thresholds
+ const isExpensive = trace.summary.cost >= strategy.minCostThreshold
+ const isLong = trace.summary.duration >= strategy.minDurationThreshold
+ const hasErrors = trace.summary.errorCount > 0
+ const isRandom = Math.random() < strategy.randomSamplingRate
+
+ const shouldAsk = isExpensive || isLong || hasErrors || isRandom
+
+ if (shouldAsk) {
+ log.debug("feedback criteria met", {
+ traceID: trace.id,
+ isExpensive,
+ isLong,
+ hasErrors,
+ isRandom,
+ })
+ }
+
+ return shouldAsk
+ }
+
+ /**
+ * Handle an idle session by checking recent traces.
+ */
+ async function handleIdleSession(sessionID: string) {
+ if (!enabled) return
+
+ log.debug("checking for feedback opportunity", { sessionID })
+
+ // Get recent traces for this session
+ const recentTraces = await getRecentTracesForSession(sessionID, 5)
+
+ if (recentTraces.length === 0) {
+ log.debug("no recent traces for feedback", { sessionID })
+ return
+ }
+
+ // Find traces worth asking about
+ const candidateTraces = recentTraces.filter((trace) =>
+ shouldRequestFeedback(trace, sessionID)
+ )
+
+ if (candidateTraces.length === 0) {
+ log.debug("no candidate traces for feedback", { sessionID })
+ return
+ }
+
+ // Request feedback for the most recent candidates (up to 3)
+ const tracesToAsk = candidateTraces.slice(0, 3)
+ await Telemetry.requestFeedback(tracesToAsk.map((t) => t.id))
+
+ // Update rate limiting
+ requestsThisHour++
+ lastRequestTime.set(sessionID, Date.now())
+
+ log.info("feedback requested", {
+ sessionID,
+ traceCount: tracesToAsk.length,
+ traceIDs: tracesToAsk.map((t) => t.id),
+ })
+ }
+
+ /**
+ * Get recent traces for a session.
+ */
+ async function getRecentTracesForSession(
+ sessionID: string,
+ limit: number
+ ): Promise {
+ try {
+ const session = await Session.get(sessionID)
+ const keys = await Storage.list(["trace", session.projectID, sessionID])
+
+ // Get all traces and sort by timestamp
+ const traces: Trace.Complete[] = []
+ for (const key of keys) {
+ try {
+ const trace = await Storage.read(key)
+ traces.push(trace)
+ } catch {
+ // Skip invalid traces
+ }
+ }
+
+ // Sort by completion time (newest first) and limit
+ return traces
+ .filter((t) => t.completedAt) // Only completed traces
+ .sort((a, b) => (b.completedAt || 0) - (a.completedAt || 0))
+ .slice(0, limit)
+ } catch (error) {
+ log.warn("failed to get recent traces", { sessionID, error })
+ return []
+ }
+ }
+
+ /**
+ * Get feedback statistics.
+ */
+ export async function getStatistics(): Promise<{
+ totalRequested: number
+ totalResponded: number
+ responseRate: number
+ avgResponseTime: number
+ }> {
+ const feedbackKeys = await Storage.list(["feedback"])
+ const feedbacks: Telemetry.UserFeedback[] = []
+
+ for (const key of feedbackKeys) {
+ try {
+ const feedback = await Storage.read(key)
+ feedbacks.push(feedback)
+ } catch {
+ // Skip invalid feedback
+ }
+ }
+
+ const totalResponded = feedbacks.length
+ // Note: We don't currently track requests separately, so this is an approximation
+ const totalRequested = totalResponded * 3 // Assume ~30% response rate
+
+ const responseRate = totalRequested > 0 ? totalResponded / totalRequested : 0
+
+ const responseTimes = feedbacks.map((f) => f.respondedAt - f.requestedAt)
+ const avgResponseTime =
+ responseTimes.length > 0
+ ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length
+ : 0
+
+ return {
+ totalRequested,
+ totalResponded,
+ responseRate,
+ avgResponseTime,
+ }
+ }
+
+ /**
+ * Update feedback strategy based on observed response rates.
+ */
+ export async function adaptStrategy() {
+ const stats = await getStatistics()
+
+ // If response rate is very low, reduce request frequency
+ if (stats.responseRate < 0.1 && stats.totalResponded > 10) {
+ strategy.maxRequestsPerHour = Math.max(1, strategy.maxRequestsPerHour - 1)
+ strategy.minTimeSinceLastRequest += 30 * 60 * 1000 // Add 30 minutes
+ log.info("adapted strategy: reduced request frequency", { strategy })
+ }
+
+ // If response rate is high, we can ask more often
+ if (stats.responseRate > 0.4 && stats.totalResponded > 20) {
+ strategy.maxRequestsPerHour = Math.min(3, strategy.maxRequestsPerHour + 1)
+ strategy.minTimeSinceLastRequest = Math.max(
+ 30 * 60 * 1000,
+ strategy.minTimeSinceLastRequest - 15 * 60 * 1000
+ )
+ log.info("adapted strategy: increased request frequency", { strategy })
+ }
+ }
+}
diff --git a/packages/opencode/src/evaluation/heuristics.ts b/packages/opencode/src/evaluation/heuristics.ts
new file mode 100644
index 0000000000..80bba7ce4a
--- /dev/null
+++ b/packages/opencode/src/evaluation/heuristics.ts
@@ -0,0 +1,319 @@
+import type { Trace } from "../trace"
+
+/**
+ * A heuristic function that evaluates a trace and returns a numeric score.
+ *
+ * @param trace - The completed trace to evaluate
+ * @param params - Optional parameters for the heuristic function
+ * @returns A numeric score representing the evaluation result
+ */
+export type HeuristicFunction = (trace: Trace.Complete, params?: Record) => number
+
+/**
+ * Built-in heuristic functions for trace evaluation.
+ *
+ * Each function analyzes different aspects of trace execution:
+ * - Performance: responseDuration, averageToolDuration, slowToolCalls
+ * - Reliability: toolErrorRate, toolSuccessRate, hasErrors
+ * - Efficiency: costEfficiency, tokenEfficiency, cacheHitRate
+ * - Usage: toolCallCount, toolUsageCount, redundantCalls
+ * - Cost: totalCost
+ *
+ * @example
+ * ```typescript
+ * const errorRate = Heuristics.toolErrorRate(trace)
+ * const slowCalls = Heuristics.slowToolCalls(trace, { threshold: 3000 })
+ * ```
+ */
+export const Heuristics = {
+ /**
+ * Calculate the ratio of failed tool calls.
+ *
+ * Returns the proportion of tool calls that ended in error status.
+ * Useful for measuring reliability and detecting integration issues.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Error rate between 0 (no errors) and 1 (all errors)
+ *
+ * @example
+ * ```typescript
+ * const errorRate = Heuristics.toolErrorRate(trace)
+ * // 0.25 means 25% of tool calls failed
+ * ```
+ */
+ toolErrorRate(trace: Trace.Complete, _params?: Record): number {
+ if (trace.toolCalls.length === 0) return 0
+ const errors = trace.toolCalls.filter((t) => t.status === "error").length
+ return errors / trace.toolCalls.length
+ },
+
+ /**
+ * Calculate the total duration in milliseconds.
+ *
+ * Measures the end-to-end execution time of the trace from start to finish.
+ * Lower values indicate better performance.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Duration in milliseconds
+ *
+ * @example
+ * ```typescript
+ * const duration = Heuristics.responseDuration(trace)
+ * // 1500 means the trace took 1.5 seconds
+ * ```
+ */
+ responseDuration(trace: Trace.Complete, _params?: Record): number {
+ return trace.summary.duration
+ },
+
+ /**
+ * Detect redundant or duplicate tool calls.
+ *
+ * Identifies tools that were called multiple times with the same parameters,
+ * which may indicate inefficient agent behavior or retry logic.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Count of tools that were called more than once with identical parameters
+ *
+ * @example
+ * ```typescript
+ * const redundant = Heuristics.redundantCalls(trace)
+ * // 2 means two different tools were called redundantly
+ * ```
+ */
+ redundantCalls(trace: Trace.Complete, _params?: Record): number {
+ const seen = new Map()
+
+ for (const call of trace.toolCalls) {
+ // Create a key from tool ID and params
+ const key = `${call.id}:${JSON.stringify(call.extra || {})}`
+ seen.set(key, (seen.get(key) || 0) + 1)
+ }
+
+ // Count how many tools were called multiple times
+ return Array.from(seen.values()).filter((count) => count > 1).length
+ },
+
+ /**
+ * Calculate cost efficiency (cost per successful operation).
+ *
+ * Measures how much each successful tool call costs on average.
+ * Lower values indicate better cost efficiency.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Cost per successful operation in dollars, or Infinity if no successful calls
+ *
+ * @example
+ * ```typescript
+ * const efficiency = Heuristics.costEfficiency(trace)
+ * // 0.01 means each successful operation costs $0.01 on average
+ * ```
+ */
+ costEfficiency(trace: Trace.Complete, _params?: Record): number {
+ const successfulCalls = trace.toolCalls.filter((t) => t.status === "success").length
+ if (successfulCalls === 0) return Infinity
+ return trace.summary.cost / successfulCalls
+ },
+
+ /**
+ * Calculate token efficiency (output tokens / total tokens).
+ *
+ * Measures the ratio of output tokens to total tokens used.
+ * Higher values indicate more productive token usage.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Ratio between 0 and 1 representing output token efficiency
+ *
+ * @example
+ * ```typescript
+ * const efficiency = Heuristics.tokenEfficiency(trace)
+ * // 0.33 means 33% of tokens were output (rest were input/reasoning)
+ * ```
+ */
+ tokenEfficiency(trace: Trace.Complete, _params?: Record): number {
+ const total =
+ trace.summary.tokens.input +
+ trace.summary.tokens.output +
+ trace.summary.tokens.reasoning
+ if (total === 0) return 0
+ return trace.summary.tokens.output / total
+ },
+
+ /**
+ * Calculate average tool call duration.
+ *
+ * Computes the mean execution time across all tool calls.
+ * Useful for understanding overall tool performance.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Average duration in milliseconds, or 0 if no tool calls
+ *
+ * @example
+ * ```typescript
+ * const avgDuration = Heuristics.averageToolDuration(trace)
+ * // 250 means tool calls took 250ms on average
+ * ```
+ */
+ averageToolDuration(trace: Trace.Complete, _params?: Record): number {
+ if (trace.toolCalls.length === 0) return 0
+ const totalDuration = trace.toolCalls.reduce((sum, call) => sum + call.duration, 0)
+ return totalDuration / trace.toolCalls.length
+ },
+
+ /**
+ * Check if any tool call exceeded a duration threshold.
+ *
+ * Counts the number of tool calls that took longer than the specified threshold.
+ * Useful for identifying performance bottlenecks.
+ *
+ * @param trace - The trace to analyze
+ * @param params - Configuration object
+ * @param params.threshold - Maximum acceptable duration in milliseconds (default: 5000)
+ * @returns Count of tool calls exceeding the threshold
+ *
+ * @example
+ * ```typescript
+ * const slow = Heuristics.slowToolCalls(trace, { threshold: 3000 })
+ * // 3 means three tool calls took longer than 3 seconds
+ * ```
+ */
+ slowToolCalls(trace: Trace.Complete, params?: { threshold?: number }): number {
+ const threshold = params?.threshold ?? 5000 // 5 seconds default
+ return trace.toolCalls.filter((t) => t.duration > threshold).length
+ },
+
+ /**
+ * Calculate the ratio of tool calls that were successful.
+ *
+ * Measures the proportion of tool calls that completed successfully.
+ * Higher values indicate better reliability.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Success rate between 0 (all failed) and 1 (all succeeded)
+ *
+ * @example
+ * ```typescript
+ * const successRate = Heuristics.toolSuccessRate(trace)
+ * // 0.95 means 95% of tool calls succeeded
+ * ```
+ */
+ toolSuccessRate(trace: Trace.Complete, _params?: Record): number {
+ if (trace.toolCalls.length === 0) return 1 // No tools = perfect success
+ const successes = trace.toolCalls.filter((t) => t.status === "success").length
+ return successes / trace.toolCalls.length
+ },
+
+ /**
+ * Count total number of tool calls.
+ *
+ * Returns the total number of tool invocations in the trace.
+ * Useful for monitoring agent activity levels.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Total count of tool calls
+ *
+ * @example
+ * ```typescript
+ * const count = Heuristics.toolCallCount(trace)
+ * // 7 means the agent made 7 tool calls
+ * ```
+ */
+ toolCallCount(trace: Trace.Complete, _params?: Record): number {
+ return trace.toolCalls.length
+ },
+
+ /**
+ * Calculate cache hit rate.
+ *
+ * Measures the proportion of input tokens that were served from cache.
+ * Higher values indicate better cache utilization and cost savings.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Cache hit rate between 0 (no cache hits) and 1 (all from cache)
+ *
+ * @example
+ * ```typescript
+ * const hitRate = Heuristics.cacheHitRate(trace)
+ * // 0.4 means 40% of input tokens came from cache
+ * ```
+ */
+ cacheHitRate(trace: Trace.Complete, _params?: Record): number {
+ const cacheRead = trace.summary.tokens.cache.read
+ const totalInput = trace.summary.tokens.input + cacheRead
+ if (totalInput === 0) return 0
+ return cacheRead / totalInput
+ },
+
+ /**
+ * Calculate total cost.
+ *
+ * Returns the total monetary cost of the trace execution.
+ * Includes all LLM API calls and token usage.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns Total cost in dollars
+ *
+ * @example
+ * ```typescript
+ * const cost = Heuristics.totalCost(trace)
+ * // 0.02 means the trace cost $0.02 to execute
+ * ```
+ */
+ totalCost(trace: Trace.Complete, _params?: Record): number {
+ return trace.summary.cost
+ },
+
+ /**
+ * Check if trace has any errors.
+ *
+ * Returns a binary indicator of whether the trace encountered any errors.
+ * Useful for pass/fail quality gates.
+ *
+ * @param trace - The trace to analyze
+ * @param _params - Unused, present for signature consistency
+ * @returns 1 if errors occurred, 0 if no errors
+ *
+ * @example
+ * ```typescript
+ * const hasErrors = Heuristics.hasErrors(trace)
+ * // 0 means the trace executed without errors
+ * ```
+ */
+ hasErrors(trace: Trace.Complete, _params?: Record): number {
+ return trace.summary.errorCount > 0 ? 1 : 0
+ },
+
+ /**
+ * Count specific tool usage.
+ *
+ * Counts how many times a particular tool was invoked during trace execution.
+ * Useful for monitoring tool usage patterns and detecting overuse.
+ *
+ * @param trace - The trace to analyze
+ * @param params - Configuration object
+ * @param params.toolId - The ID of the tool to count
+ * @returns Number of times the specified tool was called (0 if toolId not provided)
+ *
+ * @example
+ * ```typescript
+ * const readCount = Heuristics.toolUsageCount(trace, { toolId: "Read" })
+ * // 5 means the Read tool was called 5 times
+ * ```
+ */
+ toolUsageCount(trace: Trace.Complete, params?: { toolId?: string }): number {
+ if (!params?.toolId) return 0
+ return trace.toolCalls.filter((t) => t.id === params.toolId).length
+ },
+} as const
+
+export type HeuristicName = keyof typeof Heuristics
diff --git a/packages/opencode/src/evaluation/index.ts b/packages/opencode/src/evaluation/index.ts
new file mode 100644
index 0000000000..aab9bd97f5
--- /dev/null
+++ b/packages/opencode/src/evaluation/index.ts
@@ -0,0 +1,28 @@
+/**
+ * Evaluation framework for assessing trace quality
+ *
+ * This module provides:
+ * - Trace materialization from sessions
+ * - Metric definitions and registry
+ * - Evaluation engine to run metrics against traces
+ * - Built-in heuristics for common quality checks
+ * - Dataset management for test cases
+ * - Test runner for executing and validating test cases
+ * - Baseline tracking for regression detection
+ * - Time-series analysis for trend monitoring
+ * - Integration layer for automatic evaluation and alerting
+ */
+
+export { Trace } from "../trace"
+export { Metric } from "./metric"
+export { EvaluationEngine } from "./engine"
+export { Heuristics } from "./heuristics"
+export { BuiltinMetrics, registerBuiltinMetrics } from "./metrics/builtin"
+export { Dataset } from "./dataset"
+export { TestRunner } from "./runner"
+export { Baseline } from "./baseline"
+export { TimeSeries } from "./timeseries"
+export { EvaluationIntegration } from "./integration"
+export { Telemetry } from "./telemetry"
+export { FeedbackManager } from "./feedback-manager"
+export { initEvaluation } from "./init"
diff --git a/packages/opencode/src/evaluation/init.ts b/packages/opencode/src/evaluation/init.ts
new file mode 100644
index 0000000000..b5aa9380e0
--- /dev/null
+++ b/packages/opencode/src/evaluation/init.ts
@@ -0,0 +1,30 @@
+import { registerBuiltinMetrics } from "./metrics/builtin"
+import { Log } from "../util/log"
+
+const log = Log.create({ service: "evaluation-init" })
+
+let initialized = false
+
+/**
+ * Initialize the evaluation framework
+ * Registers built-in metrics and sets up event listeners
+ */
+export async function initEvaluation(): Promise {
+ if (initialized) return
+
+ try {
+ // Register all built-in metrics
+ await registerBuiltinMetrics()
+
+ log.info("evaluation framework initialized", {
+ metricsRegistered: true,
+ })
+
+ initialized = true
+ } catch (error) {
+ log.error("failed to initialize evaluation framework", {
+ error: error instanceof Error ? error.message : String(error),
+ })
+ // Don't throw - evaluation framework initialization failures shouldn't block the app
+ }
+}
diff --git a/packages/opencode/src/evaluation/integration.ts b/packages/opencode/src/evaluation/integration.ts
new file mode 100644
index 0000000000..a890b7f371
--- /dev/null
+++ b/packages/opencode/src/evaluation/integration.ts
@@ -0,0 +1,411 @@
+import { Bus } from "../bus"
+import { Log } from "../util/log"
+import { Trace } from "../trace"
+import { Baseline } from "./baseline"
+import { TimeSeries } from "./timeseries"
+import { EvaluationEngine } from "./engine"
+import { Metric } from "./metric"
+
+/**
+ * Integration layer that connects evaluation, baseline tracking, and time-series
+ * analysis with the trace lifecycle.
+ *
+ * Features:
+ * - Automatic evaluation and time-series recording on trace completion
+ * - Automatic baseline comparison for registered baselines
+ * - Alert generation for regressions and anomalies
+ * - Dashboard data aggregation
+ *
+ * @example
+ * ```typescript
+ * // Enable auto-evaluation for all completed traces
+ * await EvaluationIntegration.enableAutoEvaluation({
+ * metricIDs: ["error-rate", "latency", "cost"],
+ * recordTimeSeries: true,
+ * checkBaselines: true,
+ * })
+ *
+ * // Monitor for regressions
+ * EvaluationIntegration.onRegression((alert) => {
+ * console.log(`Regression detected: ${alert.metricID}`)
+ * notifyTeam(alert)
+ * })
+ * ```
+ */
+export namespace EvaluationIntegration {
+ const log = Log.create({ service: "evaluation-integration" })
+
+ export type Config = {
+ /** Metrics to automatically evaluate on trace completion */
+ metricIDs: string[]
+ /** Whether to record results in time-series */
+ recordTimeSeries?: boolean
+ /** Whether to compare against active baselines */
+ checkBaselines?: boolean
+ /** Tags to add to time-series data points */
+ tags?: Record
+ /** Whether to emit alerts for anomalies */
+ detectAnomalies?: boolean
+ /** Anomaly detection threshold (sigma) */
+ anomalyThreshold?: number
+ /** Whether to collect telemetry data (default: true) */
+ collectTelemetry?: boolean
+ }
+
+ export type RegressionAlert = {
+ type: "regression"
+ traceID: string
+ metricID: string
+ baselineID: string
+ baselineValue: number
+ currentValue: number
+ delta: number
+ percentChange: number
+ timestamp: number
+ }
+
+ export type AnomalyAlert = {
+ type: "anomaly"
+ traceID: string
+ metricID: string
+ currentValue: number
+ expectedRange: { min: number; max: number }
+ zScore: number
+ timestamp: number
+ }
+
+ export type ImprovementAlert = {
+ type: "improvement"
+ traceID: string
+ metricID: string
+ baselineID: string
+ baselineValue: number
+ currentValue: number
+ delta: number
+ percentChange: number
+ timestamp: number
+ }
+
+ export type Alert = RegressionAlert | AnomalyAlert | ImprovementAlert
+
+ let config: Config | null = null
+ let unsubscribe: (() => void) | null = null
+ const alertCallbacks = new Set<(alert: Alert) => void>()
+
+ /**
+ * Enable automatic evaluation and monitoring.
+ *
+ * When enabled, traces will automatically be evaluated against specified
+ * metrics, results recorded in time-series, and compared against baselines.
+ *
+ * @param cfg - Configuration for auto-evaluation
+ */
+ export async function enableAutoEvaluation(cfg: Config) {
+ if (unsubscribe) {
+ log.warn("auto-evaluation already enabled, reconfiguring")
+ unsubscribe()
+ }
+
+ config = cfg
+ log.info("enabling auto-evaluation", { metricIDs: cfg.metricIDs })
+
+ // Subscribe to trace completion events
+ unsubscribe = Bus.subscribe(Trace.Event.Completed, async ({ properties }) => {
+ try {
+ await processTrace(properties.trace, cfg)
+ } catch (error) {
+ log.error("failed to process trace", { error, traceID: properties.trace.id })
+ }
+ })
+ }
+
+ /**
+ * Disable automatic evaluation.
+ */
+ export function disableAutoEvaluation() {
+ if (unsubscribe) {
+ unsubscribe()
+ unsubscribe = null
+ config = null
+ log.info("auto-evaluation disabled")
+ }
+ }
+
+ /**
+ * Register a callback for alert notifications.
+ *
+ * @param callback - Function to call when alerts are generated
+ * @returns Unsubscribe function
+ */
+ export function onAlert(callback: (alert: Alert) => void): () => void {
+ alertCallbacks.add(callback)
+ return () => alertCallbacks.delete(callback)
+ }
+
+ /**
+ * Convenience method for regression-only alerts.
+ */
+ export function onRegression(callback: (alert: RegressionAlert) => void): () => void {
+ return onAlert((alert) => {
+ if (alert.type === "regression") callback(alert)
+ })
+ }
+
+ /**
+ * Convenience method for anomaly-only alerts.
+ */
+ export function onAnomaly(callback: (alert: AnomalyAlert) => void): () => void {
+ return onAlert((alert) => {
+ if (alert.type === "anomaly") callback(alert)
+ })
+ }
+
+ /**
+ * Convenience method for improvement-only alerts.
+ */
+ export function onImprovement(callback: (alert: ImprovementAlert) => void): () => void {
+ return onAlert((alert) => {
+ if (alert.type === "improvement") callback(alert)
+ })
+ }
+
+ /**
+ * Get dashboard data aggregating evaluation results.
+ *
+ * @param options - Filtering and aggregation options
+ * @returns Dashboard data with metrics, trends, and alerts
+ */
+ export async function getDashboard(options: {
+ since?: number
+ until?: number
+ metricIDs?: string[]
+ period?: "hour" | "day" | "week" | "month"
+ }) {
+ const metricIDs = options.metricIDs ?? config?.metricIDs ?? []
+ const period = options.period ?? "day"
+
+ const metrics = await Promise.all(
+ metricIDs.map(async (metricID) => {
+ const metric = await Metric.get(metricID)
+
+ // Get time-series data
+ const points = await TimeSeries.getDataPoints(metricID, {
+ since: options.since,
+ until: options.until,
+ })
+
+ // Get aggregates
+ const aggregates = await TimeSeries.getAggregates(metricID, { period })
+
+ // Get trend analysis
+ let trend = null
+ try {
+ const days = options.since
+ ? Math.ceil((Date.now() - options.since) / (24 * 60 * 60 * 1000))
+ : 7
+ trend = await TimeSeries.analyzeTrend(metricID, { days })
+ } catch {
+ // Not enough data
+ }
+
+ // Get associated baselines
+ const baselines = await Baseline.list()
+ const relevantBaselines = baselines.filter((b) => b.metricIDs.includes(metricID))
+
+ return {
+ metric,
+ dataPoints: points.length,
+ aggregates: aggregates.slice(-10), // Last 10 periods
+ trend,
+ baselines: relevantBaselines.map((b) => ({
+ id: b.id,
+ name: b.name,
+ statistics: b.statistics.find((s) => s.metricID === metricID),
+ })),
+ }
+ }),
+ )
+
+ return {
+ metrics,
+ period: {
+ start: options.since ?? Date.now() - 7 * 24 * 60 * 60 * 1000,
+ end: options.until ?? Date.now(),
+ },
+ }
+ }
+
+ /**
+ * Process a completed trace through the evaluation pipeline.
+ */
+ async function processTrace(trace: Trace.Complete, cfg: Config) {
+ log.debug("processing trace", { traceID: trace.id })
+
+ // 0. Enrich trace with telemetry (non-blocking)
+ if (cfg.collectTelemetry !== false) {
+ const { Telemetry } = await import("./telemetry")
+ Telemetry.enrichTrace(trace).catch((error) => {
+ log.warn("telemetry enrichment failed", { traceID: trace.id, error })
+ })
+ }
+
+ // 1. Evaluate all configured metrics
+ const metrics = await Promise.all(cfg.metricIDs.map((id) => Metric.get(id)))
+ const results = await EvaluationEngine.evaluateMany(trace, metrics)
+
+ log.debug("evaluated trace", {
+ traceID: trace.id,
+ resultsCount: results.length,
+ })
+
+ // 2. Check for anomalies if enabled (BEFORE recording to time-series)
+ if (cfg.detectAnomalies) {
+ for (const result of results) {
+ try {
+ const anomalyResult = await TimeSeries.detectAnomaly(
+ result.metricID,
+ result.score,
+ 7, // 7 days lookback
+ )
+
+ if (anomalyResult.isAnomaly) {
+ const alert: AnomalyAlert = {
+ type: "anomaly",
+ traceID: trace.id,
+ metricID: result.metricID,
+ currentValue: result.score,
+ expectedRange: anomalyResult.expectedRange,
+ zScore: anomalyResult.zScore,
+ timestamp: Date.now(),
+ }
+ emitAlert(alert)
+ }
+ } catch {
+ // Not enough data for anomaly detection
+ }
+ }
+ }
+
+ // 3. Record in time-series if enabled (AFTER anomaly detection)
+ if (cfg.recordTimeSeries) {
+ for (const result of results) {
+ await TimeSeries.record(result.metricID, trace, cfg.tags)
+ }
+ log.debug("recorded time-series", { traceID: trace.id })
+ }
+
+ // 4. Compare against baselines if enabled
+ if (cfg.checkBaselines) {
+ const baselines = await Baseline.list()
+
+ for (const baseline of baselines) {
+ // Check if this baseline applies to this trace
+ const relevantMetrics = cfg.metricIDs.filter((id) => baseline.metricIDs.includes(id))
+ if (relevantMetrics.length === 0) continue
+
+ // Skip if baseline doesn't have enough samples yet
+ if (baseline.traceIDs.length < baseline.minSampleSize) continue
+
+ try {
+ const comparison = await Baseline.compare(baseline.id, trace)
+
+ // Emit alerts for regressions
+ for (const metricID of comparison.regressions) {
+ const metricComparison = comparison.metrics.find((m) => m.metricID === metricID)
+ if (!metricComparison) continue
+
+ const alert: RegressionAlert = {
+ type: "regression",
+ traceID: trace.id,
+ metricID,
+ baselineID: baseline.id,
+ baselineValue: metricComparison.baselineValue,
+ currentValue: metricComparison.traceValue,
+ delta: metricComparison.delta,
+ percentChange: metricComparison.percentChange,
+ timestamp: Date.now(),
+ }
+ emitAlert(alert)
+ }
+
+ // Emit alerts for improvements
+ for (const metricID of comparison.improvements) {
+ const metricComparison = comparison.metrics.find((m) => m.metricID === metricID)
+ if (!metricComparison) continue
+
+ const alert: ImprovementAlert = {
+ type: "improvement",
+ traceID: trace.id,
+ metricID,
+ baselineID: baseline.id,
+ baselineValue: metricComparison.baselineValue,
+ currentValue: metricComparison.traceValue,
+ delta: metricComparison.delta,
+ percentChange: metricComparison.percentChange,
+ timestamp: Date.now(),
+ }
+ emitAlert(alert)
+ }
+ } catch (error) {
+ log.error("baseline comparison failed", { error, baselineID: baseline.id })
+ }
+ }
+ }
+
+ log.debug("trace processing complete", { traceID: trace.id })
+ }
+
+ /**
+ * Emit an alert to all registered callbacks.
+ */
+ function emitAlert(alert: Alert) {
+ log.info("emitting alert", { type: alert.type, traceID: alert.traceID })
+ for (const callback of alertCallbacks) {
+ try {
+ callback(alert)
+ } catch (error) {
+ log.error("alert callback failed", { error })
+ }
+ }
+ }
+
+ /**
+ * Manually trigger evaluation for a specific trace.
+ *
+ * Useful for re-evaluating historical traces or evaluating traces
+ * that were completed before auto-evaluation was enabled.
+ *
+ * @param traceOrID - The trace object or trace ID to evaluate
+ * @param cfg - Optional configuration (uses global config if not provided)
+ */
+ export async function evaluateTrace(traceOrID: string | Trace.Complete, cfg?: Config) {
+ const trace = typeof traceOrID === "string" ? await Trace.get(traceOrID) : traceOrID
+ const evalConfig = cfg ?? config
+ if (!evalConfig) {
+ throw new Error("No configuration provided and auto-evaluation not enabled")
+ }
+ await processTrace(trace, evalConfig)
+ }
+
+ /**
+ * Batch evaluate multiple traces.
+ *
+ * @param tracesOrIDs - Array of trace objects or trace IDs to evaluate
+ * @param cfg - Optional configuration
+ */
+ export async function evaluateTraces(tracesOrIDs: (string | Trace.Complete)[], cfg?: Config) {
+ const evalConfig = cfg ?? config
+ if (!evalConfig) {
+ throw new Error("No configuration provided and auto-evaluation not enabled")
+ }
+
+ for (const traceOrID of tracesOrIDs) {
+ try {
+ await evaluateTrace(traceOrID, evalConfig)
+ } catch (error) {
+ const id = typeof traceOrID === "string" ? traceOrID : traceOrID.id
+ log.error("failed to evaluate trace", { error, traceID: id })
+ }
+ }
+ }
+}
diff --git a/packages/opencode/src/evaluation/metric-semantics.ts b/packages/opencode/src/evaluation/metric-semantics.ts
new file mode 100644
index 0000000000..d2a8079499
--- /dev/null
+++ b/packages/opencode/src/evaluation/metric-semantics.ts
@@ -0,0 +1,354 @@
+/**
+ * Metric semantics and validation utilities.
+ *
+ * Provides type-safe semantic definitions for common metric types
+ * and validation to catch configuration errors early.
+ */
+
+import type { Metric } from "./metric"
+
+export namespace MetricSemantics {
+ /**
+ * Semantic metadata for a metric.
+ */
+ export interface Semantics {
+ unit?: string
+ interpretSlope?: (slope: number, higherIsBetter: boolean) => string
+ formatValue?: (value: number) => string
+ }
+
+ /**
+ * Common semantic patterns for standard metric types.
+ */
+ export const Common = {
+ /**
+ * Cost metrics (dollars, credits, tokens).
+ * Lower is better.
+ */
+ cost: {
+ unit: "dollars",
+ interpretSlope: (slope: number) =>
+ slope > 0 ? "increasing (worse)" : "decreasing (better)",
+ formatValue: (v: number) => `$${v.toFixed(4)}`,
+ } as Semantics,
+
+ /**
+ * Duration/latency metrics (milliseconds, seconds).
+ * Lower is better.
+ */
+ duration: {
+ unit: "milliseconds",
+ interpretSlope: (slope: number) =>
+ slope > 0 ? "slowing down (worse)" : "speeding up (better)",
+ formatValue: (v: number) => {
+ if (v < 1000) return `${v.toFixed(0)}ms`
+ return `${(v / 1000).toFixed(2)}s`
+ },
+ } as Semantics,
+
+ /**
+ * Error rate metrics (proportion, percentage).
+ * Lower is better.
+ */
+ errorRate: {
+ unit: "percent",
+ interpretSlope: (slope: number) =>
+ slope > 0 ? "more errors (worse)" : "fewer errors (better)",
+ formatValue: (v: number) => `${(v * 100).toFixed(1)}%`,
+ } as Semantics,
+
+ /**
+ * Throughput metrics (requests/second, items/second).
+ * Higher is better.
+ */
+ throughput: {
+ unit: "requests/second",
+ interpretSlope: (slope: number) =>
+ slope > 0 ? "increasing (better)" : "decreasing (worse)",
+ formatValue: (v: number) => `${v.toFixed(1)} req/s`,
+ } as Semantics,
+
+ /**
+ * Quality/accuracy metrics (score, rating).
+ * Higher is better.
+ */
+ quality: {
+ unit: "score",
+ interpretSlope: (slope: number) =>
+ slope > 0 ? "improving (better)" : "degrading (worse)",
+ formatValue: (v: number) => v.toFixed(2),
+ } as Semantics,
+
+ /**
+ * Token count metrics.
+ * Context-dependent (lower usually better for cost).
+ */
+ tokens: {
+ unit: "tokens",
+ interpretSlope: (slope: number, higherIsBetter: boolean) =>
+ higherIsBetter
+ ? slope > 0
+ ? "increasing (better)"
+ : "decreasing (worse)"
+ : slope > 0
+ ? "increasing (worse)"
+ : "decreasing (better)",
+ formatValue: (v: number) => `${Math.round(v)} tokens`,
+ } as Semantics,
+ }
+
+ /**
+ * Validation result for a metric definition.
+ */
+ export interface ValidationResult {
+ valid: boolean
+ errors: string[]
+ warnings: string[]
+ }
+
+ /**
+ * Validate metric definition for common issues.
+ *
+ * Checks for:
+ * - Semantic mismatches (cost with higherIsBetter=true)
+ * - Missing required fields
+ * - Inconsistent configuration
+ *
+ * @param metric - The metric definition to validate
+ * @returns Validation result with errors and warnings
+ *
+ * @example
+ * ```typescript
+ * const metric: Metric.Definition = {
+ * id: 'cost',
+ * evaluator: { type: 'heuristic', function: 'totalCost' },
+ * higherIsBetter: true, // WRONG!
+ * semantics: MetricSemantics.Common.cost
+ * }
+ *
+ * const result = MetricSemantics.validate(metric)
+ * // result.errors = ['Cost metrics should have higherIsBetter=false']
+ * ```
+ */
+ export function validate(metric: Metric.Definition): ValidationResult {
+ const errors: string[] = []
+ const warnings: string[] = []
+
+ // Check semantic/direction mismatches
+ if (metric.semantics?.unit === "dollars" && metric.higherIsBetter) {
+ errors.push(
+ `Metric "${metric.name}" (${metric.id}): Cost metrics should typically have higherIsBetter=false`
+ )
+ }
+
+ if (metric.semantics?.unit === "milliseconds" && metric.higherIsBetter) {
+ errors.push(
+ `Metric "${metric.name}" (${metric.id}): Duration metrics should typically have higherIsBetter=false`
+ )
+ }
+
+ if (metric.semantics?.unit === "percent" && metric.higherIsBetter) {
+ // Percent could be error rate (lower better) or success rate (higher better)
+ // Only warn if it's explicitly an error rate
+ if (
+ metric.name.toLowerCase().includes("error") ||
+ metric.id.toLowerCase().includes("error")
+ ) {
+ errors.push(
+ `Metric "${metric.name}" (${metric.id}): Error rate metrics should have higherIsBetter=false`
+ )
+ }
+ }
+
+ if (
+ (metric.semantics?.unit === "requests/second" ||
+ metric.semantics?.unit === "score") &&
+ !metric.higherIsBetter
+ ) {
+ warnings.push(
+ `Metric "${metric.name}" (${metric.id}): ${metric.semantics.unit} metrics usually have higherIsBetter=true`
+ )
+ }
+
+ // Check for missing metadata
+ if (!metric.description) {
+ warnings.push(
+ `Metric "${metric.name}" (${metric.id}): Missing description - add documentation for clarity`
+ )
+ }
+
+ if (!metric.semantics) {
+ warnings.push(
+ `Metric "${metric.name}" (${metric.id}): No semantics defined - consider adding for better formatting`
+ )
+ }
+
+ // Check category makes sense
+ if (metric.category === "cost" && metric.higherIsBetter) {
+ errors.push(
+ `Metric "${metric.name}" (${metric.id}): Category "cost" implies higherIsBetter=false`
+ )
+ }
+
+ if (metric.category === "performance" && metric.higherIsBetter === undefined) {
+ warnings.push(
+ `Metric "${metric.name}" (${metric.id}): Performance metrics should explicitly set higherIsBetter`
+ )
+ }
+
+ return {
+ valid: errors.length === 0,
+ errors,
+ warnings,
+ }
+ }
+
+ /**
+ * Suggest appropriate semantics based on metric properties.
+ *
+ * @param metric - The metric definition
+ * @returns Suggested semantics object or null
+ *
+ * @example
+ * ```typescript
+ * const metric = {
+ * id: 'response-time',
+ * category: 'performance',
+ * higherIsBetter: false
+ * }
+ *
+ * const semantics = MetricSemantics.suggest(metric)
+ * // semantics = Common.duration
+ * ```
+ */
+ export function suggest(
+ metric: Pick
+ ): Semantics | null {
+ const name = metric.name?.toLowerCase() || ""
+ const id = metric.id.toLowerCase()
+ const text = `${name} ${id}`
+
+ // Cost-related
+ if (
+ metric.category === "cost" ||
+ text.includes("cost") ||
+ text.includes("price") ||
+ text.includes("dollar")
+ ) {
+ return Common.cost
+ }
+
+ // Duration-related
+ if (
+ text.includes("duration") ||
+ text.includes("latency") ||
+ text.includes("time") ||
+ text.includes("delay")
+ ) {
+ return Common.duration
+ }
+
+ // Error-related
+ if (text.includes("error") || text.includes("failure") || text.includes("fail")) {
+ return Common.errorRate
+ }
+
+ // Throughput-related
+ if (
+ text.includes("throughput") ||
+ text.includes("rate") ||
+ text.includes("rps") ||
+ text.includes("qps")
+ ) {
+ return Common.throughput
+ }
+
+ // Quality-related
+ if (
+ text.includes("quality") ||
+ text.includes("score") ||
+ text.includes("accuracy") ||
+ text.includes("precision")
+ ) {
+ return Common.quality
+ }
+
+ // Token-related
+ if (text.includes("token")) {
+ return Common.tokens
+ }
+
+ return null
+ }
+
+ /**
+ * Format a metric value using its semantics.
+ *
+ * @param value - The value to format
+ * @param metric - The metric definition (or just semantics)
+ * @returns Formatted string
+ *
+ * @example
+ * ```typescript
+ * formatValue(0.0245, { semantics: Common.cost })
+ * // "$0.0245"
+ *
+ * formatValue(1500, { semantics: Common.duration })
+ * // "1.50s"
+ * ```
+ */
+ export function formatValue(
+ value: number,
+ metric: { semantics?: Semantics }
+ ): string {
+ if (metric.semantics?.formatValue) {
+ return metric.semantics.formatValue(value)
+ }
+
+ // Default formatting
+ if (Math.abs(value) < 0.01) {
+ return value.toExponential(2)
+ }
+ if (Math.abs(value) < 1) {
+ return value.toFixed(4)
+ }
+ if (Math.abs(value) < 100) {
+ return value.toFixed(2)
+ }
+ return Math.round(value).toString()
+ }
+
+ /**
+ * Interpret trend direction with semantic context.
+ *
+ * @param slope - The slope from trend analysis
+ * @param metric - The metric definition
+ * @returns Human-readable interpretation
+ *
+ * @example
+ * ```typescript
+ * interpretTrend(0.005, {
+ * higherIsBetter: false,
+ * semantics: Common.cost
+ * })
+ * // "increasing (worse)"
+ * ```
+ */
+ export function interpretTrend(
+ slope: number,
+ metric: { higherIsBetter: boolean; semantics?: Semantics }
+ ): string {
+ if (metric.semantics?.interpretSlope) {
+ return metric.semantics.interpretSlope(slope, metric.higherIsBetter)
+ }
+
+ // Default interpretation
+ const direction = slope > 0 ? "increasing" : "decreasing"
+ const isGood =
+ (slope > 0 && metric.higherIsBetter) ||
+ (slope < 0 && !metric.higherIsBetter)
+ const quality = isGood ? "better" : "worse"
+
+ return `${direction} (${quality})`
+ }
+}
diff --git a/packages/opencode/src/evaluation/metric.ts b/packages/opencode/src/evaluation/metric.ts
new file mode 100644
index 0000000000..cb65c34624
--- /dev/null
+++ b/packages/opencode/src/evaluation/metric.ts
@@ -0,0 +1,198 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+
+/**
+ * Metric management for trace evaluation.
+ *
+ * Metrics define how traces should be evaluated, including:
+ * - What to measure (via evaluator)
+ * - Success thresholds (pass/warn values)
+ * - Whether higher or lower scores are better
+ *
+ * Supports three evaluator types:
+ * - Rule: JavaScript expressions for custom logic
+ * - Heuristic: Built-in functions for common metrics
+ * - LLM: AI-powered evaluation (planned)
+ *
+ * @example
+ * ```typescript
+ * await Metric.register({
+ * id: "error-rate",
+ * name: "Error Rate",
+ * description: "Tool call error rate threshold",
+ * version: "1.0.0",
+ * category: "reliability",
+ * evaluator: { type: "heuristic", function: "toolErrorRate" },
+ * threshold: { pass: 0.05, warn: 0.02 },
+ * higherIsBetter: false,
+ * tags: ["production", "quality-gate"]
+ * })
+ * ```
+ */
+export namespace Metric {
+ export const Category = z.enum(["performance", "correctness", "safety", "cost", "quality", "reliability"])
+ export type Category = z.infer
+
+ export const RuleEvaluator = z.object({
+ type: z.literal("rule"),
+ expression: z.string(), // JavaScript expression evaluated against trace
+ })
+ export type RuleEvaluator = z.infer
+
+ export const HeuristicEvaluator = z.object({
+ type: z.literal("heuristic"),
+ function: z.string(), // Name of built-in heuristic function
+ params: z.record(z.string(), z.any()).optional(),
+ })
+ export type HeuristicEvaluator = z.infer
+
+ export const LLMEvaluator = z.object({
+ type: z.literal("llm"),
+ prompt: z.string(),
+ model: z.string(),
+ parseScore: z.string(), // Function body to parse LLM output to number
+ })
+ export type LLMEvaluator = z.infer
+
+ export const Evaluator = z.discriminatedUnion("type", [
+ RuleEvaluator,
+ HeuristicEvaluator,
+ LLMEvaluator,
+ ])
+ export type Evaluator = z.infer
+
+ export const Threshold = z.object({
+ pass: z.number(),
+ warn: z.number().optional(),
+ })
+ export type Threshold = z.infer
+
+ export const Definition = z.object({
+ id: z.string(),
+ name: z.string(),
+ description: z.string(),
+ version: z.string(),
+ category: Category,
+ evaluator: Evaluator,
+ threshold: Threshold.optional(),
+ higherIsBetter: z.boolean(),
+ tags: z.array(z.string()).default([]),
+ semantics: z
+ .object({
+ unit: z.string().optional(),
+ interpretSlope: z.function().optional(),
+ formatValue: z.function().optional(),
+ })
+ .optional(),
+ })
+ export type Definition = z.infer
+
+ /**
+ * Register a new metric definition.
+ *
+ * Stores the metric in the registry for use in evaluations.
+ * Metrics can be retrieved by ID, category, or tags.
+ *
+ * @param metric - The complete metric definition
+ *
+ * @example
+ * ```typescript
+ * await Metric.register({
+ * id: "cost-limit",
+ * name: "Cost Limit",
+ * description: "Maximum cost per trace",
+ * version: "1.0.0",
+ * category: "cost",
+ * evaluator: { type: "heuristic", function: "totalCost" },
+ * threshold: { pass: 0.10 },
+ * higherIsBetter: false,
+ * tags: ["budget"]
+ * })
+ * ```
+ */
+ export async function register(metric: Definition): Promise {
+ await Storage.write(["metric", metric.id], metric)
+ }
+
+ /**
+ * Get a metric by ID
+ */
+ export async function get(id: string): Promise {
+ const metric = await Storage.read(["metric", id])
+ return metric
+ }
+
+ /**
+ * List all registered metrics
+ */
+ export async function list(): Promise {
+ const keys = await Storage.list(["metric"])
+ const metrics: Definition[] = []
+
+ for (const key of keys) {
+ const metric = await Storage.read(key)
+ metrics.push(metric)
+ }
+
+ return metrics
+ }
+
+ /**
+ * Check if a metric exists
+ */
+ export async function exists(id: string): Promise {
+ try {
+ await get(id)
+ return true
+ } catch {
+ return false
+ }
+ }
+
+ /**
+ * Remove a metric
+ */
+ export async function remove(id: string): Promise {
+ await Storage.remove(["metric", id])
+ }
+
+ /**
+ * Find metrics by category.
+ *
+ * Retrieves all metrics that belong to a specific category.
+ * Categories help organize metrics by their evaluation focus.
+ *
+ * @param category - The category to filter by (performance, correctness, safety, cost, quality, reliability)
+ * @returns Array of metric definitions in the specified category
+ *
+ * @example
+ * ```typescript
+ * const costMetrics = await Metric.findByCategory("cost")
+ * console.log(`Found ${costMetrics.length} cost metrics`)
+ * ```
+ */
+ export async function findByCategory(category: Category): Promise {
+ const all = await list()
+ return all.filter((m) => m.category === category)
+ }
+
+ /**
+ * Find metrics by tag.
+ *
+ * Retrieves all metrics that have a specific tag.
+ * Tags allow flexible grouping and filtering of metrics.
+ *
+ * @param tag - The tag to filter by
+ * @returns Array of metric definitions with the specified tag
+ *
+ * @example
+ * ```typescript
+ * const prodMetrics = await Metric.findByTag("production")
+ * const gateMetrics = await Metric.findByTag("quality-gate")
+ * ```
+ */
+ export async function findByTag(tag: string): Promise {
+ const all = await list()
+ return all.filter((m) => m.tags.includes(tag))
+ }
+}
diff --git a/packages/opencode/src/evaluation/metrics/builtin.ts b/packages/opencode/src/evaluation/metrics/builtin.ts
new file mode 100644
index 0000000000..ab2b93d6d3
--- /dev/null
+++ b/packages/opencode/src/evaluation/metrics/builtin.ts
@@ -0,0 +1,197 @@
+import type { Metric } from "../metric"
+
+/**
+ * Built-in metrics available out of the box
+ */
+export const BuiltinMetrics: Record = {
+ "tool-error-rate": {
+ id: "tool-error-rate",
+ name: "Tool Error Rate",
+ description: "Percentage of tool calls that failed",
+ version: "1.0.0",
+ category: "performance",
+ evaluator: {
+ type: "heuristic",
+ function: "toolErrorRate",
+ },
+ threshold: {
+ pass: 0.1, // <10% errors is acceptable
+ warn: 0.05, // <5% is good
+ },
+ higherIsBetter: false,
+ tags: ["reliability", "tools"],
+ },
+
+ "response-latency": {
+ id: "response-latency",
+ name: "Response Latency",
+ description: "Total time to complete the request in milliseconds",
+ version: "1.0.0",
+ category: "performance",
+ evaluator: {
+ type: "heuristic",
+ function: "responseDuration",
+ },
+ threshold: {
+ pass: 30000, // <30s is acceptable
+ warn: 10000, // <10s is good
+ },
+ higherIsBetter: false,
+ tags: ["performance", "latency"],
+ },
+
+ "redundant-calls": {
+ id: "redundant-calls",
+ name: "Redundant Tool Calls",
+ description: "Number of duplicate/redundant tool calls detected",
+ version: "1.0.0",
+ category: "correctness",
+ evaluator: {
+ type: "heuristic",
+ function: "redundantCalls",
+ },
+ threshold: {
+ pass: 0, // No redundant calls
+ },
+ higherIsBetter: false,
+ tags: ["efficiency", "tools"],
+ },
+
+ "cost-efficiency": {
+ id: "cost-efficiency",
+ name: "Cost Efficiency",
+ description: "Cost per successful tool operation",
+ version: "1.0.0",
+ category: "cost",
+ evaluator: {
+ type: "heuristic",
+ function: "costEfficiency",
+ },
+ threshold: {
+ pass: 0.05, // <$0.05 per operation
+ warn: 0.01, // <$0.01 is good
+ },
+ higherIsBetter: false,
+ tags: ["cost", "efficiency"],
+ },
+
+ "token-efficiency": {
+ id: "token-efficiency",
+ name: "Token Efficiency",
+ description: "Ratio of output tokens to total tokens used",
+ version: "1.0.0",
+ category: "cost",
+ evaluator: {
+ type: "heuristic",
+ function: "tokenEfficiency",
+ },
+ threshold: {
+ pass: 0.2, // At least 20% of tokens are output
+ warn: 0.3, // 30%+ is good
+ },
+ higherIsBetter: true,
+ tags: ["cost", "efficiency"],
+ },
+
+ "average-tool-duration": {
+ id: "average-tool-duration",
+ name: "Average Tool Duration",
+ description: "Average time per tool call in milliseconds",
+ version: "1.0.0",
+ category: "performance",
+ evaluator: {
+ type: "heuristic",
+ function: "averageToolDuration",
+ },
+ threshold: {
+ pass: 3000, // <3s average
+ warn: 1000, // <1s is good
+ },
+ higherIsBetter: false,
+ tags: ["performance", "tools"],
+ },
+
+ "tool-success-rate": {
+ id: "tool-success-rate",
+ name: "Tool Success Rate",
+ description: "Percentage of tool calls that succeeded",
+ version: "1.0.0",
+ category: "reliability",
+ evaluator: {
+ type: "heuristic",
+ function: "toolSuccessRate",
+ },
+ threshold: {
+ pass: 0.9, // >90% success
+ warn: 0.95, // >95% is good
+ },
+ higherIsBetter: true,
+ tags: ["reliability", "tools"],
+ },
+
+ "cache-hit-rate": {
+ id: "cache-hit-rate",
+ name: "Cache Hit Rate",
+ description: "Percentage of input tokens served from cache",
+ version: "1.0.0",
+ category: "cost",
+ evaluator: {
+ type: "heuristic",
+ function: "cacheHitRate",
+ },
+ threshold: {
+ pass: 0.3, // >30% cache hits
+ warn: 0.5, // >50% is good
+ },
+ higherIsBetter: true,
+ tags: ["cost", "performance"],
+ },
+
+ "total-cost": {
+ id: "total-cost",
+ name: "Total Cost",
+ description: "Total cost of the trace in dollars",
+ version: "1.0.0",
+ category: "cost",
+ evaluator: {
+ type: "heuristic",
+ function: "totalCost",
+ },
+ threshold: {
+ pass: 1.0, // <$1 per trace
+ warn: 0.1, // <$0.10 is good
+ },
+ higherIsBetter: false,
+ tags: ["cost"],
+ },
+
+ "has-errors": {
+ id: "has-errors",
+ name: "Has Errors",
+ description: "Whether the trace encountered any errors",
+ version: "1.0.0",
+ category: "reliability",
+ evaluator: {
+ type: "heuristic",
+ function: "hasErrors",
+ },
+ threshold: {
+ pass: 0, // No errors
+ },
+ higherIsBetter: false,
+ tags: ["reliability"],
+ },
+}
+
+/**
+ * Register all built-in metrics
+ */
+export async function registerBuiltinMetrics(): Promise {
+ const { Metric } = await import("../metric")
+
+ for (const metric of Object.values(BuiltinMetrics)) {
+ if (!(await Metric.exists(metric.id))) {
+ await Metric.register(metric)
+ }
+ }
+}
diff --git a/packages/opencode/src/evaluation/runner.ts b/packages/opencode/src/evaluation/runner.ts
new file mode 100644
index 0000000000..a0a3d28d6b
--- /dev/null
+++ b/packages/opencode/src/evaluation/runner.ts
@@ -0,0 +1,412 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import { Bus } from "../bus"
+import { Log } from "../util/log"
+import type { Trace } from "../trace"
+import { Dataset } from "./dataset"
+import { EvaluationEngine } from "./engine"
+
+/**
+ * TestRunner executes test suites and validates trace behavior.
+ *
+ * The runner evaluates assertions against traces to determine if they
+ * meet expected criteria. It supports:
+ * - Running entire datasets of test cases
+ * - Evaluating individual assertions against traces
+ * - Tracking test history and results
+ * - Emitting events for test lifecycle monitoring
+ *
+ * Assertion results include pass/fail status, actual vs expected values,
+ * and descriptive messages for debugging failures.
+ *
+ * @example
+ * ```typescript
+ * // Run assertions against a trace
+ * const assertions = [
+ * { type: "tool-called", toolID: "Read", minCount: 1 },
+ * { type: "duration-under", milliseconds: 5000 },
+ * { type: "no-errors" }
+ * ]
+ * const results = await TestRunner.runAssertions(trace, assertions)
+ * const passed = results.every(r => r.passed)
+ * ```
+ */
+export namespace TestRunner {
+ const log = Log.create({ service: "test-runner" })
+
+ export const AssertionResult = z.object({
+ assertion: Dataset.Assertion,
+ passed: z.boolean(),
+ message: z.string(),
+ actual: z.any().optional(),
+ expected: z.any().optional(),
+ })
+ export type AssertionResult = z.infer
+
+ export const TestResult = z.object({
+ testCase: Dataset.TestCase,
+ traceID: z.string(),
+ passed: z.boolean(),
+
+ assertions: z.array(AssertionResult),
+
+ duration: z.number(),
+ timestamp: z.number(),
+
+ error: z.string().optional(),
+ })
+ export type TestResult = z.infer
+
+ export const RunResult = z.object({
+ id: z.string(),
+ datasetID: z.string(),
+
+ results: z.array(TestResult),
+
+ summary: z.object({
+ total: z.number(),
+ passed: z.number(),
+ failed: z.number(),
+ duration: z.number(),
+ }),
+
+ timestamp: z.number(),
+ })
+ export type RunResult = z.infer
+
+ export const Event = {
+ Started: Bus.event(
+ "test.started",
+ z.object({
+ runID: z.string(),
+ datasetID: z.string(),
+ }),
+ ),
+ TestCompleted: Bus.event(
+ "test.completed",
+ z.object({
+ runID: z.string(),
+ testCaseID: z.string(),
+ passed: z.boolean(),
+ }),
+ ),
+ Completed: Bus.event(
+ "test.run.completed",
+ z.object({
+ runID: z.string(),
+ summary: RunResult.shape.summary,
+ }),
+ ),
+ }
+
+ /**
+ * Run all test cases in a dataset
+ */
+ export async function run(datasetID: string): Promise {
+ const { Dataset } = await import("./dataset")
+ const dataset = await Dataset.get(datasetID)
+ const testCases = dataset.testCases.filter((tc) => tc.enabled)
+
+ const runID = Date.now().toString() + "-" + Math.random().toString(36).substring(7)
+ const startTime = Date.now()
+
+ Bus.publish(Event.Started, { runID, datasetID })
+
+ log.info("starting test run", {
+ runID,
+ datasetID,
+ testCount: testCases.length,
+ })
+
+ const results: TestResult[] = []
+
+ for (const testCase of testCases) {
+ const result = await runTest(testCase, runID)
+ results.push(result)
+
+ Bus.publish(Event.TestCompleted, {
+ runID,
+ testCaseID: testCase.id,
+ passed: result.passed,
+ })
+
+ log.info("test completed", {
+ testCaseID: testCase.id,
+ passed: result.passed,
+ assertions: result.assertions.length,
+ })
+ }
+
+ const endTime = Date.now()
+ const passed = results.filter((r) => r.passed).length
+
+ const runResult: RunResult = {
+ id: runID,
+ datasetID,
+ results,
+ summary: {
+ total: results.length,
+ passed,
+ failed: results.length - passed,
+ duration: endTime - startTime,
+ },
+ timestamp: startTime,
+ }
+
+ // Store the run result
+ await Storage.write(["test-run", datasetID, runID], runResult)
+
+ Bus.publish(Event.Completed, {
+ runID,
+ summary: runResult.summary,
+ })
+
+ log.info("test run completed", {
+ runID,
+ summary: runResult.summary,
+ })
+
+ return runResult
+ }
+
+ /**
+ * Run a single test case
+ */
+ async function runTest(testCase: Dataset.TestCase, _runID: string): Promise {
+ const startTime = Date.now()
+
+ try {
+ // For now, we need a trace to evaluate assertions
+ // In a full implementation, this would execute the agent with the test input
+ // and create a new trace. For now, we'll document this limitation.
+
+ // TODO: Implement agent execution here
+ // const trace = await executeAgent(testCase.input.prompt, testCase.input.context)
+
+ // Placeholder: We'll need to provide a way to link test cases to existing traces
+ // or execute the agent to create new traces
+ throw new Error("Test execution requires agent integration - not yet implemented")
+
+ } catch (error) {
+ return {
+ testCase,
+ traceID: "",
+ passed: false,
+ assertions: [],
+ duration: Date.now() - startTime,
+ timestamp: startTime,
+ error: error instanceof Error ? error.message : String(error),
+ }
+ }
+ }
+
+ /**
+ * Run assertions against a trace.
+ *
+ * Evaluates all provided assertions and returns results with
+ * pass/fail status, actual vs expected values, and messages.
+ *
+ * @param trace - The completed trace to validate
+ * @param assertions - Array of assertions to evaluate
+ * @returns Array of assertion results with pass/fail status
+ *
+ * @example
+ * ```typescript
+ * const assertions = [
+ * { type: "tool-called", toolID: "Edit", minCount: 1, maxCount: 3 },
+ * { type: "output-contains", substring: "success" },
+ * { type: "cost-under", dollars: 0.05 }
+ * ]
+ * const results = await TestRunner.runAssertions(trace, assertions)
+ * results.forEach(r => {
+ * console.log(`${r.passed ? '✓' : '✗'} ${r.message}`)
+ * })
+ * ```
+ */
+ export async function runAssertions(trace: Trace.Complete, assertions: Dataset.Assertion[]): Promise {
+ return Promise.all(assertions.map((assertion) => checkAssertion(trace, assertion)))
+ }
+
+ /**
+ * Check a single assertion
+ */
+ async function checkAssertion(trace: Trace.Complete, assertion: Dataset.Assertion): Promise {
+ try {
+ switch (assertion.type) {
+ case "tool-called":
+ return checkToolCalled(trace, assertion)
+ case "output-matches":
+ return checkOutputMatches(trace, assertion)
+ case "output-contains":
+ return checkOutputContains(trace, assertion)
+ case "no-errors":
+ return checkNoErrors(trace)
+ case "duration-under":
+ return checkDurationUnder(trace, assertion)
+ case "cost-under":
+ return checkCostUnder(trace, assertion)
+ case "metric-passes":
+ return checkMetricPasses(trace, assertion)
+ case "custom":
+ return checkCustom(trace, assertion)
+ }
+ } catch (error) {
+ return {
+ assertion,
+ passed: false,
+ message: `Assertion check failed: ${error instanceof Error ? error.message : String(error)}`,
+ }
+ }
+ }
+
+ function checkToolCalled(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "tool-called" }): AssertionResult {
+ const calls = trace.toolCalls.filter((tc) => tc.id === assertion.toolID)
+ const count = calls.length
+
+ const minCount = assertion.minCount ?? 1
+ const maxCount = assertion.maxCount ?? Infinity
+
+ const passed = count >= minCount && count <= maxCount
+
+ return {
+ assertion,
+ passed,
+ message: passed
+ ? `Tool '${assertion.toolID}' called ${count} time(s)`
+ : `Tool '${assertion.toolID}' called ${count} time(s), expected ${minCount} to ${maxCount === Infinity ? "∞" : maxCount}`,
+ actual: count,
+ expected: { min: minCount, max: maxCount },
+ }
+ }
+
+ function checkOutputMatches(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "output-matches" }): AssertionResult {
+ const output = trace.output
+
+ const regex = new RegExp(assertion.pattern, assertion.flags)
+ const passed = regex.test(output)
+
+ return {
+ assertion,
+ passed,
+ message: passed
+ ? `Output matches pattern: ${assertion.pattern}`
+ : `Output does not match pattern: ${assertion.pattern}`,
+ actual: output,
+ expected: assertion.pattern,
+ }
+ }
+
+ function checkOutputContains(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "output-contains" }): AssertionResult {
+ const output = trace.output
+
+ const passed = output.includes(assertion.substring)
+
+ return {
+ assertion,
+ passed,
+ message: passed
+ ? `Output contains: "${assertion.substring}"`
+ : `Output does not contain: "${assertion.substring}"`,
+ actual: output,
+ expected: assertion.substring,
+ }
+ }
+
+ function checkNoErrors(trace: Trace.Complete): AssertionResult {
+ const passed = trace.summary.errorCount === 0
+
+ return {
+ assertion: { type: "no-errors" },
+ passed,
+ message: passed ? "No errors" : `Found ${trace.summary.errorCount} error(s)`,
+ actual: trace.summary.errorCount,
+ expected: 0,
+ }
+ }
+
+ function checkDurationUnder(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "duration-under" }): AssertionResult {
+ const passed = trace.summary.duration <= assertion.milliseconds
+
+ return {
+ assertion,
+ passed,
+ message: passed
+ ? `Duration ${trace.summary.duration}ms under ${assertion.milliseconds}ms`
+ : `Duration ${trace.summary.duration}ms exceeds ${assertion.milliseconds}ms`,
+ actual: trace.summary.duration,
+ expected: assertion.milliseconds,
+ }
+ }
+
+ function checkCostUnder(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "cost-under" }): AssertionResult {
+ const passed = trace.summary.cost <= assertion.dollars
+
+ return {
+ assertion,
+ passed,
+ message: passed
+ ? `Cost $${trace.summary.cost.toFixed(4)} under $${assertion.dollars}`
+ : `Cost $${trace.summary.cost.toFixed(4)} exceeds $${assertion.dollars}`,
+ actual: trace.summary.cost,
+ expected: assertion.dollars,
+ }
+ }
+
+ async function checkMetricPasses(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "metric-passes" }): Promise {
+ const { Metric } = await import("./metric")
+ const metric = await Metric.get(assertion.metricID)
+ const result = await EvaluationEngine.evaluate(trace, metric)
+
+ return {
+ assertion,
+ passed: result.passed,
+ message: result.passed
+ ? `Metric '${metric.name}' passed with score ${result.score}`
+ : `Metric '${metric.name}' failed with score ${result.score}`,
+ actual: result.score,
+ expected: metric.threshold,
+ }
+ }
+
+ function checkCustom(trace: Trace.Complete, assertion: Dataset.Assertion & { type: "custom" }): AssertionResult {
+ try {
+ const func = new Function("trace", `return ${assertion.expression}`)
+ const result = func(trace)
+ const passed = Boolean(result)
+
+ return {
+ assertion,
+ passed,
+ message: passed ? assertion.description : `${assertion.description} (failed)`,
+ }
+ } catch (error) {
+ return {
+ assertion,
+ passed: false,
+ message: `Custom assertion failed: ${error instanceof Error ? error.message : String(error)}`,
+ }
+ }
+ }
+
+ /**
+ * Get test run history for a dataset
+ */
+ export async function getRunHistory(datasetID: string): Promise {
+ const keys = await Storage.list(["test-run", datasetID])
+ const results: RunResult[] = []
+
+ for (const key of keys) {
+ const result = await Storage.read(key)
+ results.push(result)
+ }
+
+ return results.sort((a, b) => b.timestamp - a.timestamp)
+ }
+
+ /**
+ * Get a specific test run
+ */
+ export async function getRun(datasetID: string, runID: string): Promise {
+ return Storage.read(["test-run", datasetID, runID])
+ }
+}
diff --git a/packages/opencode/src/evaluation/telemetry.ts b/packages/opencode/src/evaluation/telemetry.ts
new file mode 100644
index 0000000000..b1ff5e672f
--- /dev/null
+++ b/packages/opencode/src/evaluation/telemetry.ts
@@ -0,0 +1,581 @@
+/**
+ * Telemetry collection for production insights.
+ *
+ * Enriches traces with contextual metadata to enable:
+ * - Cost attribution and analysis
+ * - Quality prediction without user input
+ * - Performance segmentation
+ * - Self-improving evaluation
+ *
+ * Key principles:
+ * - Non-invasive: Doesn't modify core trace schema
+ * - Opt-in: Only collected when evaluation is enabled
+ * - Async: Never blocks user operations
+ * - Privacy-preserving: Metadata only, no code content
+ */
+
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import { Bus } from "../bus"
+import type { Trace } from "../trace"
+import { Instance } from "../project/instance"
+import { Log } from "../util/log"
+import { FileWatcher } from "../file/watcher"
+
+const log = Log.create({ service: "evaluation.telemetry" })
+
+export namespace Telemetry {
+ /**
+ * Codebase characteristics for context.
+ */
+ export const CodebaseContext = z.object({
+ size: z.object({
+ files: z.number(),
+ lines: z.number(),
+ }),
+ primaryLanguage: z.string(), // Most common file extension
+ architecture: z.enum(["monolith", "microservices", "unknown"]),
+ testCoverage: z.number().optional(), // If detectable
+ })
+ export type CodebaseContext = z.infer
+
+ /**
+ * Task classification based on trace characteristics.
+ */
+ export const TaskClassification = z.object({
+ type: z.enum(["edit", "refactor", "debug", "review", "explore", "unknown"]),
+ complexity: z.enum(["simple", "medium", "complex"]),
+ confidence: z.number().min(0).max(1), // How confident are we?
+ })
+ export type TaskClassification = z.infer
+
+ /**
+ * Outcome proxies (quality signals without user feedback).
+ */
+ export const OutcomeProxies = z.object({
+ subsequentEdits: z.number().default(0), // Edits to same files within 1 hour
+ subsequentEditWindow: z.number().default(60 * 60 * 1000), // 1 hour in ms
+ gitReverted: z.boolean().optional(), // Was change reverted?
+ testResults: z
+ .object({
+ total: z.number(),
+ passed: z.number(),
+ failed: z.number(),
+ })
+ .optional(),
+ })
+ export type OutcomeProxies = z.infer
+
+ /**
+ * Enriched trace metadata.
+ */
+ export const EnrichedMetadata = z.object({
+ traceID: z.string(),
+ timestamp: z.number(),
+
+ // Context
+ codebaseContext: CodebaseContext.optional(),
+ taskClassification: TaskClassification,
+
+ // Attribution
+ userEmail: z.string().optional(),
+ teamID: z.string().optional(),
+ environment: z.enum(["development", "production", "staging"]).optional(),
+
+ // Outcome tracking
+ outcomeProxies: OutcomeProxies,
+
+ // Metadata
+ collectedAt: z.number(),
+ version: z.string().default("1.0.0"),
+ })
+ export type EnrichedMetadata = z.infer
+
+ /**
+ * User feedback for specific traces.
+ */
+ export const UserFeedback = z.object({
+ traceID: z.string(),
+ timestamp: z.number(),
+
+ // Structured questions
+ responses: z.object({
+ correctness: z.number().min(1).max(5).optional(), // 1-5 rating
+ speed: z.enum(["too-slow", "acceptable", "fast"]).optional(),
+ wouldUseAgain: z.boolean().optional(),
+ }),
+
+ // Freeform
+ comment: z.string().optional(),
+
+ // Context
+ requestedAt: z.number(),
+ respondedAt: z.number(),
+ })
+ export type UserFeedback = z.infer
+
+ /**
+ * Feedback request configuration.
+ */
+ export const FeedbackRequest = z.object({
+ traceIDs: z.array(z.string()),
+ questions: z.array(
+ z.object({
+ id: z.string(),
+ type: z.enum(["rating", "choice", "boolean", "text"]),
+ question: z.string(),
+ options: z.array(z.string()).optional(),
+ })
+ ),
+ requestedAt: z.number(),
+ })
+ export type FeedbackRequest = z.infer
+
+ /**
+ * Events for telemetry system.
+ */
+ export const Event = {
+ Enriched: Bus.event(
+ "telemetry.enriched",
+ z.object({
+ metadata: EnrichedMetadata,
+ })
+ ),
+ FeedbackRequested: Bus.event(
+ "telemetry.feedback_requested",
+ z.object({
+ request: FeedbackRequest,
+ })
+ ),
+ FeedbackReceived: Bus.event(
+ "telemetry.feedback_received",
+ z.object({
+ feedback: UserFeedback,
+ })
+ ),
+ }
+
+ // Track file edits for outcome detection (reserved for future use)
+ // const recentEdits = new Map>()
+
+ /**
+ * Classify task type based on trace characteristics.
+ */
+ function classifyTask(trace: Trace.Complete): TaskClassification {
+ const { toolCalls, summary } = trace
+
+ // Count tool types
+ const toolTypes = new Map()
+ for (const call of toolCalls) {
+ // Tool name can be in 'tool' property or 'id' property (from realistic traces)
+ const name = (call as any).tool || (call as any).id || "unknown"
+ toolTypes.set(name, (toolTypes.get(name) || 0) + 1)
+ }
+
+ // Heuristics for task type
+ let type: TaskClassification["type"] = "unknown"
+ let confidence = 0.5
+
+ // Explore: Mostly Read/Grep
+ const readCount = (toolTypes.get("Read") || 0) + (toolTypes.get("Grep") || 0)
+ const editCount =
+ (toolTypes.get("Edit") || 0) +
+ (toolTypes.get("MultiEdit") || 0) +
+ (toolTypes.get("Create") || 0)
+ const executeCount = toolTypes.get("Execute") || 0
+
+ if (readCount > editCount * 3 && editCount < 2) {
+ type = "explore"
+ confidence = 0.8
+ }
+ // Debug: Errors with retries (any execute + errors)
+ else if (summary.errorCount > 0 && executeCount > 0) {
+ type = "debug"
+ confidence = 0.75
+ }
+ // Refactor: MultiEdit or many edits
+ else if (toolTypes.has("MultiEdit") || editCount > 3) {
+ type = "refactor"
+ confidence = 0.7
+ }
+ // Edit: Some edits with reads
+ else if (editCount > 0 && editCount <= 3) {
+ type = "edit"
+ confidence = 0.7
+ }
+ // Review: Reads with no edits
+ else if (readCount > 0 && editCount === 0) {
+ type = "review"
+ confidence = 0.6
+ }
+
+ // Determine complexity
+ let complexity: TaskClassification["complexity"] = "medium"
+ const totalTools = summary.toolCallCount
+ const duration = summary.duration
+
+ if (totalTools <= 3 && duration < 5000 && summary.cost < 0.01) {
+ complexity = "simple"
+ } else if (totalTools > 10 || duration > 30000 || summary.cost > 0.15) {
+ complexity = "complex"
+ }
+
+ return { type, complexity, confidence }
+ }
+
+ /**
+ * Analyze codebase to extract context.
+ * Cached per project to avoid repeated scans.
+ */
+ const codebaseContextCache = new Map()
+
+ async function getCodebaseContext(projectID: string): Promise {
+ // Check cache
+ if (codebaseContextCache.has(projectID)) {
+ return codebaseContextCache.get(projectID)
+ }
+
+ try {
+ const worktree = Instance.worktree
+
+ // Skip codebase analysis if worktree is too small (test environment)
+ const fileList = await Bun.$`find ${worktree} -type f -not -path "*/node_modules/*" -not -path "*/.git/*" -not -path "*/dist/*" -not -path "*/build/*" 2>/dev/null | head -100 || echo ""`.text()
+ const files = fileList.trim().split("\n").filter(Boolean)
+
+ // If less than 3 files, it's likely a test environment
+ if (files.length < 3) {
+ return undefined
+ }
+ const fileCount = files.length
+
+ // Sample 100 files to estimate total lines
+ const sampleSize = Math.min(100, files.length)
+ const sampleFiles = files.slice(0, sampleSize)
+ let sampleLines = 0
+
+ for (const file of sampleFiles) {
+ try {
+ const content = await Bun.file(file).text()
+ sampleLines += content.split("\n").length
+ } catch {
+ // Skip files that can't be read
+ }
+ }
+
+ const estimatedLines = Math.round((sampleLines / sampleSize) * fileCount)
+
+ // Detect primary language
+ const extensions = new Map()
+ for (const file of files) {
+ const ext = file.split(".").pop()?.toLowerCase() || ""
+ if (ext) {
+ extensions.set(ext, (extensions.get(ext) || 0) + 1)
+ }
+ }
+
+ const primaryExt =
+ Array.from(extensions.entries()).sort((a, b) => b[1] - a[1])[0]?.[0] || "unknown"
+
+ const languageMap: Record = {
+ ts: "typescript",
+ js: "javascript",
+ tsx: "typescript",
+ jsx: "javascript",
+ py: "python",
+ go: "go",
+ rs: "rust",
+ java: "java",
+ rb: "ruby",
+ php: "php",
+ c: "c",
+ cpp: "cpp",
+ cs: "csharp",
+ }
+ const primaryLanguage = languageMap[primaryExt] || primaryExt
+
+ // Detect architecture (simple heuristic)
+ const hasDockerCompose = files.some((f) => f.includes("docker-compose"))
+ const hasMultiplePackageJsons = files.filter((f) => f.endsWith("package.json")).length > 1
+ const hasServicesDir = files.some((f) => f.includes("/services/") || f.includes("/apps/"))
+
+ let architecture: CodebaseContext["architecture"] = "unknown"
+ if (hasDockerCompose || hasMultiplePackageJsons || hasServicesDir) {
+ architecture = "microservices"
+ } else if (fileCount > 10) {
+ architecture = "monolith"
+ }
+
+ const context: CodebaseContext = {
+ size: {
+ files: fileCount,
+ lines: estimatedLines,
+ },
+ primaryLanguage,
+ architecture,
+ }
+
+ // Cache for 1 hour
+ codebaseContextCache.set(projectID, context)
+ setTimeout(() => codebaseContextCache.delete(projectID), 60 * 60 * 1000)
+
+ return context
+ } catch (error) {
+ log.warn("failed to analyze codebase", { error: String(error) })
+ return undefined
+ }
+ }
+
+ /**
+ * Track subsequent edits to the same files.
+ */
+ function trackSubsequentEdits(traceID: string, trace: Trace.Complete) {
+ // Extract files touched in this trace
+ const filesEdited = new Set()
+ for (const call of trace.toolCalls) {
+ const event = call as any
+ if (
+ event.tool === "Edit" ||
+ event.tool === "MultiEdit" ||
+ event.tool === "Create" ||
+ event.tool === "Write"
+ ) {
+ const file = event.params?.file || event.params?.filepath
+ if (file) filesEdited.add(file)
+ }
+ }
+
+ if (filesEdited.size === 0) return
+
+ // Subscribe to file watcher for the next hour
+ let editCount = 0
+ const unsubscribe = Bus.subscribe(FileWatcher.Event.Updated, (event) => {
+ const { file } = event.properties
+ if (filesEdited.has(file)) {
+ editCount++
+ log.debug("detected subsequent edit", { traceID, file, editCount })
+ }
+ })
+
+ // After 1 hour, update outcome proxies
+ setTimeout(async () => {
+ unsubscribe()
+
+ try {
+ const metadata = await getEnrichedMetadata(traceID)
+ if (metadata) {
+ metadata.outcomeProxies.subsequentEdits = editCount
+ await Storage.write(["telemetry", traceID], metadata)
+ log.info("updated outcome proxies", { traceID, editCount })
+ }
+ } catch (error) {
+ log.warn("failed to update outcome proxies", { traceID, error: String(error) })
+ }
+ }, 60 * 60 * 1000) // 1 hour
+ }
+
+ /**
+ * Enrich a trace with telemetry metadata.
+ * Called automatically when trace completes.
+ */
+ export async function enrichTrace(trace: Trace.Complete): Promise {
+ log.debug("enriching trace", { traceID: trace.id })
+
+ // Classify task
+ const taskClassification = classifyTask(trace)
+
+ // Get codebase context (cached)
+ const codebaseContext = await getCodebaseContext(trace.projectID)
+
+ // Create enriched metadata
+ const metadata: EnrichedMetadata = {
+ traceID: trace.id,
+ timestamp: trace.createdAt,
+ codebaseContext,
+ taskClassification,
+ outcomeProxies: {
+ subsequentEdits: 0,
+ subsequentEditWindow: 60 * 60 * 1000,
+ },
+ collectedAt: Date.now(),
+ version: "1.0.0",
+ }
+
+ // Store metadata
+ await Storage.write(["telemetry", trace.id], metadata)
+
+ // Emit event
+ Bus.publish(Event.Enriched, { metadata })
+
+ // Start tracking subsequent edits
+ trackSubsequentEdits(trace.id, trace)
+
+ log.info("trace enriched", {
+ traceID: trace.id,
+ taskType: taskClassification.type,
+ complexity: taskClassification.complexity,
+ })
+
+ return metadata
+ }
+
+ /**
+ * Get enriched metadata for a trace.
+ */
+ export async function getEnrichedMetadata(traceID: string): Promise {
+ try {
+ const metadata = await Storage.read(["telemetry", traceID])
+ return metadata
+ } catch {
+ return null
+ }
+ }
+
+ /**
+ * Record user feedback for a trace.
+ */
+ export async function recordFeedback(feedback: UserFeedback): Promise {
+ await Storage.write(["feedback", feedback.traceID], feedback)
+ Bus.publish(Event.FeedbackReceived, { feedback })
+ log.info("feedback recorded", { traceID: feedback.traceID })
+ }
+
+ /**
+ * Get user feedback for a trace.
+ */
+ export async function getFeedback(traceID: string): Promise {
+ try {
+ const feedback = await Storage.read(["feedback", traceID])
+ return feedback
+ } catch {
+ return null
+ }
+ }
+
+ /**
+ * Request feedback for specific traces.
+ * Emits an event that UI layers can subscribe to.
+ */
+ export async function requestFeedback(traceIDs: string[]): Promise {
+ const request: FeedbackRequest = {
+ traceIDs,
+ questions: [
+ {
+ id: "correctness",
+ type: "rating",
+ question: "How would you rate the quality of the result?",
+ },
+ {
+ id: "speed",
+ type: "choice",
+ question: "Was the response time acceptable?",
+ options: ["too-slow", "acceptable", "fast"],
+ },
+ {
+ id: "wouldUseAgain",
+ type: "boolean",
+ question: "Would you use this feature again?",
+ },
+ ],
+ requestedAt: Date.now(),
+ }
+
+ Bus.publish(Event.FeedbackRequested, { request })
+ log.info("feedback requested", { traceCount: traceIDs.length })
+ }
+
+ /**
+ * Query telemetry data with filters.
+ */
+ export async function query(options: {
+ since?: number
+ until?: number
+ taskType?: TaskClassification["type"]
+ complexity?: TaskClassification["complexity"]
+ limit?: number
+ }): Promise {
+ const keys = await Storage.list(["telemetry"])
+ const results: EnrichedMetadata[] = []
+
+ for (const key of keys) {
+ try {
+ const metadata = await Storage.read(key)
+
+ // Skip invalid/incomplete entries
+ if (!metadata || !metadata.taskClassification) continue
+
+ // Apply filters
+ if (options.since && metadata.timestamp < options.since) continue
+ if (options.until && metadata.timestamp > options.until) continue
+ if (options.taskType && metadata.taskClassification.type !== options.taskType) continue
+ if (options.complexity && metadata.taskClassification.complexity !== options.complexity)
+ continue
+
+ results.push(metadata)
+
+ if (options.limit && results.length >= options.limit) break
+ } catch {
+ // Skip entries that can't be read or parsed
+ continue
+ }
+ }
+
+ return results.sort((a, b) => b.timestamp - a.timestamp)
+ }
+
+ /**
+ * Get aggregated statistics from telemetry data.
+ */
+ export async function getStatistics(options?: {
+ since?: number
+ until?: number
+ }): Promise<{
+ totalTraces: number
+ byTaskType: Record
+ byComplexity: Record
+ avgSubsequentEdits: number
+ }> {
+ const metadata = await query({ since: options?.since, until: options?.until })
+
+ const byTaskType: Record = {}
+ const byComplexity: Record = {}
+ let totalSubsequentEdits = 0
+
+ for (const m of metadata) {
+ byTaskType[m.taskClassification.type] = (byTaskType[m.taskClassification.type] || 0) + 1
+ byComplexity[m.taskClassification.complexity] =
+ (byComplexity[m.taskClassification.complexity] || 0) + 1
+ totalSubsequentEdits += m.outcomeProxies.subsequentEdits
+ }
+
+ return {
+ totalTraces: metadata.length,
+ byTaskType,
+ byComplexity,
+ avgSubsequentEdits: metadata.length > 0 ? totalSubsequentEdits / metadata.length : 0,
+ }
+ }
+
+ /**
+ * Clean up old telemetry data (>30 days).
+ */
+ export async function cleanup(maxAgeMs: number = 30 * 24 * 60 * 60 * 1000): Promise {
+ const cutoff = Date.now() - maxAgeMs
+ const keys = await Storage.list(["telemetry"])
+ let removed = 0
+
+ for (const key of keys) {
+ try {
+ const metadata = await Storage.read(key)
+ if (metadata.timestamp < cutoff) {
+ await Storage.remove(key)
+ removed++
+ }
+ } catch {
+ // Skip invalid entries
+ }
+ }
+
+ log.info("telemetry cleanup completed", { removed })
+ return removed
+ }
+}
diff --git a/packages/opencode/src/evaluation/time-utils.ts b/packages/opencode/src/evaluation/time-utils.ts
new file mode 100644
index 0000000000..421d87471d
--- /dev/null
+++ b/packages/opencode/src/evaluation/time-utils.ts
@@ -0,0 +1,220 @@
+/**
+ * Time utilities for consistent timezone handling and timestamp operations.
+ *
+ * All utilities use UTC to avoid timezone confusion.
+ */
+
+export namespace TimeUtils {
+ /**
+ * Extract hour of day in UTC.
+ * Always uses UTC to avoid timezone confusion.
+ */
+ export function getHourOfDay(timestamp: number): number {
+ return new Date(timestamp).getUTCHours()
+ }
+
+ /**
+ * Get day of week in UTC (0 = Sunday, 6 = Saturday).
+ */
+ export function getDayOfWeek(timestamp: number): number {
+ return new Date(timestamp).getUTCDay()
+ }
+
+ /**
+ * Check if timestamp falls within business hours (UTC).
+ * Default: 9am-5pm UTC (configurable)
+ */
+ export function isBusinessHours(
+ timestamp: number,
+ config = { startHour: 9, endHour: 17 }
+ ): boolean {
+ const hour = getHourOfDay(timestamp)
+ return hour >= config.startHour && hour <= config.endHour
+ }
+
+ /**
+ * Check if timestamp is a weekend (UTC).
+ */
+ export function isWeekend(timestamp: number): boolean {
+ const day = getDayOfWeek(timestamp)
+ return day === 0 || day === 6 // Sunday or Saturday
+ }
+
+ /**
+ * Create evenly-spaced timestamps for simulation/testing.
+ *
+ * @param startOrDaysAgo - Days ago (number) or specific start timestamp
+ * @param endOrNow - Specific end timestamp or Date.now()
+ * @param count - Number of timestamps to generate
+ * @returns Array of evenly-spaced timestamps
+ *
+ * @example
+ * // Create 100 timestamps spanning last 7 days
+ * const timestamps = createTimeRange(7, Date.now(), 100)
+ *
+ * // Create 50 timestamps between two specific dates
+ * const timestamps = createTimeRange(
+ * new Date('2024-01-01').getTime(),
+ * new Date('2024-01-31').getTime(),
+ * 50
+ * )
+ */
+ export function createTimeRange(
+ startOrDaysAgo: number | Date,
+ endOrNow: number | Date = Date.now(),
+ count: number
+ ): number[] {
+ if (count < 2) {
+ throw new Error('count must be at least 2')
+ }
+
+ const start = typeof startOrDaysAgo === 'number'
+ ? Date.now() - startOrDaysAgo * 24 * 60 * 60 * 1000
+ : startOrDaysAgo instanceof Date
+ ? startOrDaysAgo.getTime()
+ : startOrDaysAgo
+
+ const end = typeof endOrNow === 'number'
+ ? endOrNow
+ : endOrNow instanceof Date
+ ? endOrNow.getTime()
+ : endOrNow
+
+ if (start >= end) {
+ throw new Error('start must be before end')
+ }
+
+ const step = (end - start) / (count - 1)
+
+ return Array.from({ length: count }, (_, i) => Math.floor(start + i * step))
+ }
+
+ /**
+ * Format timestamp for human-readable debugging.
+ *
+ * @example
+ * formatTimestamp(Date.now())
+ * // "2024-01-15T10:30:00.000Z (0h ago)"
+ *
+ * formatTimestamp(Date.now() - 3600000)
+ * // "2024-01-15T09:30:00.000Z (1h ago)"
+ */
+ export function formatTimestamp(timestamp: number): string {
+ const date = new Date(timestamp)
+ const hoursAgo = Math.floor((Date.now() - timestamp) / (60 * 60 * 1000))
+
+ if (hoursAgo < 0) {
+ return `${date.toISOString()} (${Math.abs(hoursAgo)}h from now)`
+ }
+ if (hoursAgo === 0) {
+ const minutesAgo = Math.floor((Date.now() - timestamp) / (60 * 1000))
+ return `${date.toISOString()} (${minutesAgo}m ago)`
+ }
+ if (hoursAgo < 48) {
+ return `${date.toISOString()} (${hoursAgo}h ago)`
+ }
+ const daysAgo = Math.floor(hoursAgo / 24)
+ return `${date.toISOString()} (${daysAgo}d ago)`
+ }
+
+ /**
+ * Validate timestamp is reasonable.
+ * Throws if timestamp is clearly invalid.
+ * Warns if timestamp is suspiciously far from now.
+ *
+ * @param timestamp - Timestamp to validate
+ * @param context - Context for error messages
+ * @param options - Validation options
+ * @returns The validated timestamp
+ *
+ * @example
+ * validateTimestamp(trace.createdAt, 'TimeSeries.record')
+ * validateTimestamp(timestamp, 'test', { warnIfOlderThanDays: 30 })
+ */
+ export function validateTimestamp(
+ timestamp: number,
+ context: string,
+ options: {
+ warnIfOlderThanDays?: number
+ warnIfNewerThanDays?: number
+ } = {}
+ ): number {
+ // Check for obviously invalid values
+ if (!timestamp || !Number.isFinite(timestamp)) {
+ throw new Error(`Invalid timestamp in ${context}: ${timestamp}`)
+ }
+
+ if (timestamp <= 0) {
+ throw new Error(`Timestamp must be positive in ${context}: ${timestamp}`)
+ }
+
+ // Check if timestamp is in milliseconds (not seconds)
+ if (timestamp < 1000000000000) {
+ throw new Error(
+ `Timestamp appears to be in seconds, not milliseconds in ${context}: ${timestamp}. ` +
+ `Did you mean ${timestamp * 1000}?`
+ )
+ }
+
+ const now = Date.now()
+ const ageMs = now - timestamp
+ const ageDays = ageMs / (24 * 60 * 60 * 1000)
+
+ // Warn if timestamp is from the future
+ if (timestamp > now) {
+ const futureDays = -ageDays
+ const warnThreshold = options.warnIfNewerThanDays ?? 1
+
+ if (futureDays > warnThreshold) {
+ console.warn(
+ `[TimeUtils] Timestamp is ${futureDays.toFixed(1)} days in the future. ` +
+ `This might indicate a bug. Context: ${context}, Timestamp: ${formatTimestamp(timestamp)}`
+ )
+ }
+ }
+
+ // Warn if timestamp is very old
+ const oldThreshold = options.warnIfOlderThanDays ?? 365
+ if (ageDays > oldThreshold) {
+ console.warn(
+ `[TimeUtils] Timestamp is ${ageDays.toFixed(1)} days old (>${oldThreshold} days). ` +
+ `This might indicate a bug. Context: ${context}, Timestamp: ${formatTimestamp(timestamp)}`
+ )
+ }
+
+ return timestamp
+ }
+
+ /**
+ * Round timestamp to nearest hour.
+ * Useful for bucketing and aggregation.
+ */
+ export function roundToHour(timestamp: number): number {
+ return Math.floor(timestamp / (60 * 60 * 1000)) * (60 * 60 * 1000)
+ }
+
+ /**
+ * Round timestamp to nearest day.
+ */
+ export function roundToDay(timestamp: number): number {
+ return Math.floor(timestamp / (24 * 60 * 60 * 1000)) * (24 * 60 * 60 * 1000)
+ }
+
+ /**
+ * Get start of day (00:00:00.000 UTC).
+ */
+ export function startOfDay(timestamp: number): number {
+ const date = new Date(timestamp)
+ date.setUTCHours(0, 0, 0, 0)
+ return date.getTime()
+ }
+
+ /**
+ * Get end of day (23:59:59.999 UTC).
+ */
+ export function endOfDay(timestamp: number): number {
+ const date = new Date(timestamp)
+ date.setUTCHours(23, 59, 59, 999)
+ return date.getTime()
+ }
+}
diff --git a/packages/opencode/src/evaluation/timeseries.ts b/packages/opencode/src/evaluation/timeseries.ts
new file mode 100644
index 0000000000..dc8db744a3
--- /dev/null
+++ b/packages/opencode/src/evaluation/timeseries.ts
@@ -0,0 +1,694 @@
+import z from "zod/v4"
+import { Storage } from "../storage/storage"
+import type { Trace } from "../trace"
+import { EvaluationEngine } from "./engine"
+import { TimeUtils } from "./time-utils"
+
+/**
+ * Time-series analysis for tracking metric trends over time.
+ *
+ * Enables tracking of metric performance across temporal dimensions:
+ * - Hourly, daily, weekly aggregations
+ * - Trend detection (improving, degrading, stable)
+ * - Anomaly detection based on historical patterns
+ * - Rolling window statistics
+ *
+ * @example
+ * ```typescript
+ * // Track metrics over time
+ * await TimeSeries.record("error-rate", trace)
+ *
+ * // Get daily aggregates for the last 7 days
+ * const trend = await TimeSeries.getAggregates("error-rate", {
+ * period: "day",
+ * since: Date.now() - 7 * 24 * 60 * 60 * 1000
+ * })
+ *
+ * // Detect trends
+ * const analysis = await TimeSeries.analyzeTrend("error-rate", { days: 7 })
+ * if (analysis.trend === "degrading") {
+ * console.warn("Metric is degrading over time")
+ * }
+ * ```
+ */
+export namespace TimeSeries {
+ /**
+ * A single data point in a time series.
+ */
+ export const DataPoint = z.object({
+ metricID: z.string(),
+ traceID: z.string(),
+ value: z.number(),
+ timestamp: z.number(),
+
+ // Context
+ tags: z.record(z.string(), z.string()).optional(),
+ })
+ export type DataPoint = z.infer
+
+ /**
+ * Aggregated statistics for a time period.
+ */
+ export const Aggregate = z.object({
+ metricID: z.string(),
+ period: z.enum(["hour", "day", "week", "month"]),
+ periodStart: z.number(),
+ periodEnd: z.number(),
+
+ // Statistics
+ count: z.number(),
+ mean: z.number(),
+ median: z.number(),
+ min: z.number(),
+ max: z.number(),
+ stdDev: z.number(),
+ p50: z.number(),
+ p95: z.number(),
+ p99: z.number(),
+ })
+ export type Aggregate = z.infer
+
+ /**
+ * Trend analysis result.
+ */
+ export const TrendAnalysis = z.object({
+ metricID: z.string(),
+ period: z.object({
+ start: z.number(),
+ end: z.number(),
+ days: z.number(),
+ }),
+
+ // Trend direction
+ trend: z.enum(["improving", "degrading", "stable"]),
+ trendStrength: z.number(), // 0-1, how strong the trend is
+
+ // Statistical measures
+ slope: z.number(), // Rate of change per day
+ correlation: z.number(), // -1 to 1, linear correlation with time
+
+ // Data points
+ dataPoints: z.number(),
+ mean: z.number(),
+ changePercent: z.number(),
+
+ // Anomalies detected
+ anomalies: z.array(
+ z.object({
+ timestamp: z.number(),
+ value: z.number(),
+ expectedValue: z.number(),
+ deviationSigmas: z.number(),
+ }),
+ ),
+ })
+ export type TrendAnalysis = z.infer
+
+ /**
+ * Record a metric value for time-series tracking.
+ *
+ * @param metricID - The metric to track
+ * @param trace - The trace containing the metric evaluation
+ * @param tags - Optional tags for filtering/grouping
+ *
+ * @example
+ * ```typescript
+ * await TimeSeries.record("latency", trace, {
+ * environment: "production",
+ * version: "v1.2.0"
+ * })
+ * ```
+ */
+ export async function record(
+ metricID: string,
+ trace: Trace.Complete,
+ tags?: Record,
+ ): Promise {
+ const { Metric } = await import("./metric")
+ const metric = await Metric.get(metricID)
+
+ // Evaluate the metric
+ const result = await EvaluationEngine.evaluate(trace, metric)
+
+ // Validate and use trace timestamp
+ const timestamp = TimeUtils.validateTimestamp(
+ trace.createdAt || Date.now(),
+ `TimeSeries.record(${metricID}, ${trace.id})`,
+ { warnIfOlderThanDays: 90 }
+ )
+
+ const dataPoint: DataPoint = {
+ metricID,
+ traceID: trace.id,
+ value: result.score,
+ timestamp,
+ tags,
+ }
+
+ // Store in time-series bucket
+ const hourBucket = Math.floor(timestamp / (60 * 60 * 1000)) // Hourly buckets
+ await Storage.write(["timeseries", metricID, hourBucket.toString(), trace.id], dataPoint)
+ }
+
+ /**
+ * Record multiple traces efficiently in a single batch operation.
+ * Much faster than calling record() in a loop.
+ *
+ * @param metricID - The metric ID
+ * @param traces - Array of traces to record
+ * @param tags - Optional tags to apply to all data points
+ *
+ * @example
+ * ```typescript
+ * const traces = generateHistoricalTraces(100)
+ * await TimeSeries.recordBatch("cost-metric", traces)
+ * ```
+ */
+ export async function recordBatch(
+ metricID: string,
+ traces: Trace.Complete[],
+ tags?: Record
+ ): Promise {
+ const { Metric } = await import("./metric")
+ const metric = await Metric.get(metricID)
+
+ // Evaluate all traces in parallel
+ const dataPoints = await Promise.all(
+ traces.map(async (trace) => {
+ const result = await EvaluationEngine.evaluate(trace, metric)
+ const timestamp = TimeUtils.validateTimestamp(
+ trace.createdAt || Date.now(),
+ `TimeSeries.recordBatch(${metricID})`,
+ { warnIfOlderThanDays: 90 }
+ )
+
+ return {
+ metricID,
+ traceID: trace.id,
+ value: result.score,
+ timestamp,
+ tags,
+ }
+ })
+ )
+
+ // Write all data points in parallel
+ await Promise.all(
+ dataPoints.map(async (point) => {
+ const hourBucket = Math.floor(point.timestamp / (60 * 60 * 1000))
+ await Storage.write(
+ ["timeseries", metricID, hourBucket.toString(), point.traceID],
+ point
+ )
+ })
+ )
+ }
+
+ /**
+ * Get raw data points for a metric within a time range.
+ *
+ * @param metricID - The metric ID
+ * @param options - Query options
+ * @returns Array of data points
+ */
+ export async function getDataPoints(
+ metricID: string,
+ options?: {
+ since?: number
+ until?: number
+ tags?: Record
+ },
+ ): Promise {
+ const keys = await Storage.list(["timeseries", metricID])
+ const points: DataPoint[] = []
+
+ for (const key of keys) {
+ const point = await Storage.read(key)
+
+ // Filter by time range
+ if (options?.since && point.timestamp < options.since) continue
+ if (options?.until && point.timestamp > options.until) continue
+
+ // Filter by tags
+ if (options?.tags) {
+ const matchesTags = Object.entries(options.tags).every(
+ ([k, v]) => point.tags?.[k] === v,
+ )
+ if (!matchesTags) continue
+ }
+
+ points.push(point)
+ }
+
+ return points.sort((a, b) => a.timestamp - b.timestamp)
+ }
+
+ /**
+ * Get aggregated statistics for a metric by time period.
+ *
+ * @param metricID - The metric ID
+ * @param options - Aggregation options
+ * @returns Array of aggregates per period
+ *
+ * @example
+ * ```typescript
+ * // Get daily stats for last month
+ * const dailyStats = await TimeSeries.getAggregates("cost", {
+ * period: "day",
+ * since: Date.now() - 30 * 24 * 60 * 60 * 1000
+ * })
+ * ```
+ */
+ export async function getAggregates(
+ metricID: string,
+ options: {
+ period: "hour" | "day" | "week" | "month"
+ since?: number
+ until?: number
+ tags?: Record
+ },
+ ): Promise {
+ const points = await getDataPoints(metricID, {
+ since: options.since,
+ until: options.until,
+ tags: options.tags,
+ })
+
+ if (points.length === 0) {
+ return []
+ }
+
+ // Group by period
+ const periodMs = getPeriodMilliseconds(options.period)
+ const groups = new Map()
+
+ for (const point of points) {
+ const periodStart = Math.floor(point.timestamp / periodMs) * periodMs
+ if (!groups.has(periodStart)) {
+ groups.set(periodStart, [])
+ }
+ groups.get(periodStart)!.push(point)
+ }
+
+ // Compute aggregates for each period
+ const aggregates: Aggregate[] = []
+
+ for (const [periodStart, groupPoints] of groups.entries()) {
+ const values = groupPoints.map((p) => p.value).sort((a, b) => a - b)
+ const count = values.length
+
+ if (count === 0) continue
+
+ const mean = values.reduce((sum, v) => sum + v, 0) / count
+ const median = values[Math.floor(count / 2)]
+ const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / count
+ const stdDev = Math.sqrt(variance)
+
+ aggregates.push({
+ metricID,
+ period: options.period,
+ periodStart,
+ periodEnd: periodStart + periodMs,
+ count,
+ mean,
+ median,
+ min: values[0],
+ max: values[count - 1],
+ stdDev,
+ p50: values[Math.floor(count * 0.5)],
+ p95: values[Math.floor(count * 0.95)],
+ p99: values[Math.floor(count * 0.99)],
+ })
+ }
+
+ return aggregates.sort((a, b) => a.periodStart - b.periodStart)
+ }
+
+ /**
+ * Analyze trend for a metric over a time period.
+ *
+ * Performs linear regression and anomaly detection to characterize
+ * the metric's behavior over time.
+ *
+ * @param metricID - The metric ID
+ * @param options - Analysis options
+ * @returns Trend analysis with direction, strength, and anomalies
+ *
+ * @example
+ * ```typescript
+ * const analysis = await TimeSeries.analyzeTrend("error-rate", {
+ * days: 14,
+ * anomalyThreshold: 3 // 3 sigma
+ * })
+ *
+ * if (analysis.trend === "degrading" && analysis.trendStrength > 0.5) {
+ * alert("Strong degradation detected!")
+ * }
+ * ```
+ */
+ export async function analyzeTrend(
+ metricID: string,
+ options: {
+ days?: number
+ since?: number
+ until?: number
+ anomalyThreshold?: number // Sigma threshold for anomaly detection
+ skipQualityCheck?: boolean // Skip data quality checks (for testing)
+ },
+ ): Promise {
+ const { Metric } = await import("./metric")
+ const metric = await Metric.get(metricID)
+
+ // Determine time range
+ const end = options.until || Date.now()
+ const days = options.days || 7
+ const start = options.since || end - days * 24 * 60 * 60 * 1000
+
+ // Check data quality first (unless explicitly skipped)
+ if (!options.skipQualityCheck) {
+ const quality = await checkDataQuality(metricID, { since: start, until: end })
+
+ if (quality.totalPoints === 0) {
+ throw new Error(`No data points available for trend analysis`)
+ }
+
+ if (quality.warnings.length > 0) {
+ console.warn(
+ `[TimeSeries] Data quality issues for ${metricID}:\n` +
+ quality.warnings.map(w => ` - ${w}`).join('\n')
+ )
+ }
+ }
+
+ // Get data points
+ const points = await getDataPoints(metricID, { since: start, until: end })
+
+ if (points.length < 3) {
+ throw new Error(`Not enough data points for trend analysis (need at least 3, got ${points.length})`)
+ }
+
+ // Normalize timestamps to days from start
+ const values = points.map((p) => p.value)
+ const times = points.map((p) => (p.timestamp - start) / (24 * 60 * 60 * 1000))
+
+ // Linear regression
+ const n = values.length
+ const sumX = times.reduce((sum, t) => sum + t, 0)
+ const sumY = values.reduce((sum, v) => sum + v, 0)
+ const sumXY = times.reduce((sum, t, i) => sum + t * values[i], 0)
+ const sumXX = times.reduce((sum, t) => sum + t * t, 0)
+
+ const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX)
+ const intercept = (sumY - slope * sumX) / n
+
+ // Correlation coefficient
+ const meanX = sumX / n
+ const meanY = sumY / n
+ const numerator = times.reduce((sum, t, i) => sum + (t - meanX) * (values[i] - meanY), 0)
+ const denomX = Math.sqrt(times.reduce((sum, t) => sum + Math.pow(t - meanX, 2), 0))
+ const denomY = Math.sqrt(values.reduce((sum, v) => sum + Math.pow(v - meanY, 2), 0))
+ const correlation = numerator / (denomX * denomY)
+
+ // Determine trend direction based on correlation strength
+ let trend: "improving" | "degrading" | "stable"
+ const trendStrength = Math.abs(correlation)
+
+ // Use correlation threshold to determine if trend is significant
+ // Correlation > 0.5 indicates moderate to strong linear trend
+ if (trendStrength < 0.5) {
+ trend = "stable"
+ } else {
+ const isIncreasing = slope > 0
+ trend = (metric.higherIsBetter && isIncreasing) || (!metric.higherIsBetter && !isIncreasing)
+ ? "improving"
+ : "degrading"
+ }
+
+ // Anomaly detection using z-score
+ const mean = meanY
+ const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / n
+ const stdDev = Math.sqrt(variance)
+ const anomalyThreshold = options.anomalyThreshold || 3
+
+ const anomalies = []
+ for (let i = 0; i < points.length; i++) {
+ const expectedValue = intercept + slope * times[i]
+ const deviation = values[i] - expectedValue
+ const zScore = stdDev === 0 ? 0 : deviation / stdDev
+
+ if (Math.abs(zScore) > anomalyThreshold) {
+ anomalies.push({
+ timestamp: points[i].timestamp,
+ value: values[i],
+ expectedValue,
+ deviationSigmas: zScore,
+ })
+ }
+ }
+
+ // Calculate percent change from start to end
+ const startValue = intercept
+ const endValue = intercept + slope * days
+ const changePercent = startValue === 0 ? 0 : ((endValue - startValue) / startValue) * 100
+
+ return {
+ metricID,
+ period: {
+ start,
+ end,
+ days,
+ },
+ trend,
+ trendStrength,
+ slope,
+ correlation,
+ dataPoints: points.length,
+ mean,
+ changePercent,
+ anomalies,
+ }
+ }
+
+ /**
+ * Detect if current metric value is an anomaly compared to historical data.
+ *
+ * @param metricID - The metric ID
+ * @param currentValue - The current value to check
+ * @param lookbackDays - Days of history to compare against
+ * @returns Whether value is anomalous and details
+ */
+ export async function detectAnomaly(
+ metricID: string,
+ currentValue: number,
+ lookbackDays = 7,
+ ): Promise<{
+ isAnomaly: boolean
+ zScore: number
+ expectedRange: { min: number; max: number }
+ historicalMean: number
+ historicalStdDev: number
+ }> {
+ const since = Date.now() - lookbackDays * 24 * 60 * 60 * 1000
+ const points = await getDataPoints(metricID, { since })
+
+ if (points.length < 3) {
+ return {
+ isAnomaly: false,
+ zScore: 0,
+ expectedRange: { min: currentValue, max: currentValue },
+ historicalMean: currentValue,
+ historicalStdDev: 0,
+ }
+ }
+
+ const values = points.map((p) => p.value)
+ const mean = values.reduce((sum, v) => sum + v, 0) / values.length
+ const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length
+ const stdDev = Math.sqrt(variance)
+
+ // Handle edge case where all values are identical (stdDev = 0)
+ // If current value differs significantly from mean, it's an anomaly
+ let zScore = 0
+ let isAnomaly = false
+
+ if (stdDev === 0) {
+ // All historical values are identical
+ const deviation = Math.abs(currentValue - mean)
+ // If deviation is more than 10% of mean (or > 0.01 for small values), it's anomalous
+ isAnomaly = deviation > Math.max(mean * 0.1, 0.01)
+ zScore = isAnomaly ? 10 : 0 // Arbitrary large z-score
+ } else {
+ zScore = (currentValue - mean) / stdDev
+ isAnomaly = Math.abs(zScore) > 3 // 3-sigma rule
+ }
+
+ return {
+ isAnomaly,
+ zScore,
+ expectedRange: {
+ min: mean - 3 * stdDev,
+ max: mean + 3 * stdDev,
+ },
+ historicalMean: mean,
+ historicalStdDev: stdDev,
+ }
+ }
+
+ /**
+ * Clear all time-series data for a specific metric.
+ * Useful for testing and data cleanup.
+ *
+ * @param metricID - The metric ID to clear data for
+ */
+ export async function clearMetric(metricID: string): Promise {
+ const prefix = ["timeseries", metricID]
+ const keys = await Storage.list(prefix)
+
+ for (const key of keys) {
+ await Storage.remove(key)
+ }
+ }
+
+ /**
+ * Data quality report for a metric's time-series data.
+ */
+ export interface DataQualityReport {
+ totalPoints: number
+ timeRange: {
+ start: number
+ end: number
+ durationDays: number
+ }
+ gaps: Array<{
+ start: number
+ end: number
+ durationHours: number
+ }>
+ duplicates: number
+ outOfOrderPoints: number
+ warnings: string[]
+ }
+
+ /**
+ * Analyze data quality for a metric.
+ * Helps identify issues before they cause analysis failures.
+ *
+ * @param metricID - The metric ID
+ * @param options - Query options
+ * @returns Data quality report with warnings
+ *
+ * @example
+ * ```typescript
+ * const quality = await TimeSeries.checkDataQuality("cost-metric")
+ * if (quality.warnings.length > 0) {
+ * console.warn("Data issues:", quality.warnings)
+ * }
+ * ```
+ */
+ export async function checkDataQuality(
+ metricID: string,
+ options?: { since?: number; until?: number }
+ ): Promise {
+ const points = await getDataPoints(metricID, options)
+
+ if (points.length === 0) {
+ return {
+ totalPoints: 0,
+ timeRange: { start: 0, end: 0, durationDays: 0 },
+ gaps: [],
+ duplicates: 0,
+ outOfOrderPoints: 0,
+ warnings: ["No data points found"],
+ }
+ }
+
+ const sorted = [...points].sort((a, b) => a.timestamp - b.timestamp)
+ const start = sorted[0].timestamp
+ const end = sorted[sorted.length - 1].timestamp
+ const durationDays = (end - start) / (24 * 60 * 60 * 1000)
+
+ // Detect gaps (> 2 hours between points)
+ const gaps: DataQualityReport["gaps"] = []
+ for (let i = 1; i < sorted.length; i++) {
+ const gapMs = sorted[i].timestamp - sorted[i - 1].timestamp
+ const gapHours = gapMs / (60 * 60 * 1000)
+ if (gapHours > 2) {
+ gaps.push({
+ start: sorted[i - 1].timestamp,
+ end: sorted[i].timestamp,
+ durationHours: gapHours,
+ })
+ }
+ }
+
+ // Detect duplicates (same timestamp)
+ const timestamps = new Set()
+ let duplicates = 0
+ for (const point of points) {
+ if (timestamps.has(point.timestamp)) {
+ duplicates++
+ }
+ timestamps.add(point.timestamp)
+ }
+
+ // Detect out-of-order points (from original array)
+ let outOfOrderPoints = 0
+ for (let i = 0; i < points.length - 1; i++) {
+ if (points[i].timestamp > points[i + 1].timestamp) {
+ outOfOrderPoints++
+ }
+ }
+
+ // Generate warnings
+ const warnings: string[] = []
+ if (points.length < 10) {
+ warnings.push(
+ `Only ${points.length} data points - need more for reliable analysis`
+ )
+ }
+ if (durationDays < 1) {
+ warnings.push(
+ `Data spans only ${durationDays.toFixed(1)} days - trends may not be reliable`
+ )
+ }
+ if (gaps.length > 0) {
+ const largestGap = Math.max(...gaps.map((g) => g.durationHours))
+ warnings.push(
+ `${gaps.length} data gaps detected (largest: ${largestGap.toFixed(1)}h) - may affect trend analysis`
+ )
+ }
+ if (duplicates > 0) {
+ warnings.push(`${duplicates} duplicate timestamps - may skew statistics`)
+ }
+ if (outOfOrderPoints > 0) {
+ warnings.push(
+ `${outOfOrderPoints} out-of-order points - data may be corrupted`
+ )
+ }
+
+ return {
+ totalPoints: points.length,
+ timeRange: { start, end, durationDays },
+ gaps,
+ duplicates,
+ outOfOrderPoints,
+ warnings,
+ }
+ }
+
+ /**
+ * Get period duration in milliseconds.
+ */
+ function getPeriodMilliseconds(period: "hour" | "day" | "week" | "month"): number {
+ switch (period) {
+ case "hour":
+ return 60 * 60 * 1000
+ case "day":
+ return 24 * 60 * 60 * 1000
+ case "week":
+ return 7 * 24 * 60 * 60 * 1000
+ case "month":
+ return 30 * 24 * 60 * 60 * 1000
+ }
+ }
+}
diff --git a/packages/opencode/src/file/index.ts b/packages/opencode/src/file/index.ts
index e5023f0dc1..3fb351a365 100644
--- a/packages/opencode/src/file/index.ts
+++ b/packages/opencode/src/file/index.ts
@@ -195,7 +195,6 @@ export namespace File {
const content = await Bun.file(full)
.text()
.catch(() => "")
- .then((x) => x.trim())
if (project.vcs === "git") {
let diff = await $`git diff ${file}`.cwd(Instance.directory).quiet().nothrow().text()
if (!diff.trim()) diff = await $`git diff --staged ${file}`.cwd(Instance.directory).quiet().nothrow().text()
diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts
index 7a54f0b2d6..224075594f 100644
--- a/packages/opencode/src/index.ts
+++ b/packages/opencode/src/index.ts
@@ -65,6 +65,10 @@ const cli = yargs(hideBin(process.argv))
version: Installation.VERSION,
args: process.argv.slice(2),
})
+
+ // Initialize evaluation framework
+ const { initEvaluation } = await import("./evaluation/init")
+ await initEvaluation()
})
.usage("\n" + UI.logo())
.command(McpCommand)
diff --git a/packages/opencode/src/installation/index.ts b/packages/opencode/src/installation/index.ts
index b01ce5f7b2..0a27d9f9d0 100644
--- a/packages/opencode/src/installation/index.ts
+++ b/packages/opencode/src/installation/index.ts
@@ -139,7 +139,7 @@ export namespace Installation {
export const USER_AGENT = `opencode/${VERSION}`
export async function latest() {
- return fetch("https://api.github.com/repos/sst/opencode/releases/latest")
+ return fetch("https://api.github.com/repos/evalops/opencode/releases/latest")
.then((res) => res.json())
.then((data) => {
if (typeof data.tag_name !== "string") {
diff --git a/packages/opencode/src/project/instance.ts b/packages/opencode/src/project/instance.ts
index 01ea87a3ca..e56a21c14a 100644
--- a/packages/opencode/src/project/instance.ts
+++ b/packages/opencode/src/project/instance.ts
@@ -9,22 +9,27 @@ interface Context {
}
const context = Context.create("instance")
const cache = new Map()
+const pending: string[] = []
export const Instance = {
async provide(input: { directory: string; init?: () => Promise; fn: () => R }): Promise {
- let existing = cache.get(input.directory)
- if (!existing) {
- const project = await Project.fromDirectory(input.directory)
- existing = {
- directory: input.directory,
+ const dir = input.directory
+ const cached = cache.get(dir)
+ const existing = cached ?? (await (async () => {
+ pending.push(dir)
+ const project = await Project.fromDirectory(dir).finally(() => {
+ pending.pop()
+ })
+ return {
+ directory: dir,
worktree: project.worktree,
project,
}
- }
+ })())
return context.provide(existing, async () => {
- if (!cache.has(input.directory)) {
+ if (!cache.has(dir)) {
await input.init?.()
- cache.set(input.directory, existing)
+ cache.set(dir, existing)
}
return input.fn()
})
@@ -38,6 +43,9 @@ export const Instance = {
get project() {
return context.use().project
},
+ get pending() {
+ return pending[pending.length - 1]
+ },
state(init: () => S, dispose?: (state: Awaited) => Promise): () => S {
return State.create(() => Instance.directory, init, dispose)
},
diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts
index 920699a2b3..3232a5f178 100644
--- a/packages/opencode/src/provider/transform.ts
+++ b/packages/opencode/src/provider/transform.ts
@@ -92,7 +92,7 @@ export namespace ProviderTransform {
}
if (modelID.includes("gpt-5") && !modelID.includes("gpt-5-chat")) {
- if (!modelID.includes("codex")) result["reasoningEffort"] = "medium"
+ result["reasoningEffort"] = "medium"
if (providerID !== "azure") {
result["textVerbosity"] = modelID.includes("codex") ? "medium" : "low"
}
diff --git a/packages/opencode/src/session/prompt/anthropic-20250930.txt b/packages/opencode/src/session/prompt/anthropic-20250930.txt
index 7a4faea633..62971393f1 100644
--- a/packages/opencode/src/session/prompt/anthropic-20250930.txt
+++ b/packages/opencode/src/session/prompt/anthropic-20250930.txt
@@ -140,7 +140,7 @@ The user will primarily request you perform software engineering tasks. This inc
Here is useful information about the environment you are running in:
-Working directory: /home/thdxr/dev/projects/sst/opencode/packages/opencode
+Working directory: /home/thdxr/dev/projects/evalops/opencode/packages/opencode
Is directory a git repo: Yes
Platform: linux
OS Version: Linux 6.12.4-arch1-1
diff --git a/packages/opencode/src/session/prompt/anthropic.txt b/packages/opencode/src/session/prompt/anthropic.txt
index 6e623fdadd..7fd2203752 100644
--- a/packages/opencode/src/session/prompt/anthropic.txt
+++ b/packages/opencode/src/session/prompt/anthropic.txt
@@ -4,7 +4,7 @@ IMPORTANT: You must NEVER generate or guess URLs for the user unless you are con
If the user asks for help or wants to give feedback inform them of the following:
- /help: Get help with using opencode
-- To give feedback, users should report the issue at https://github.com/sst/opencode/issues
+- To give feedback, users should report the issue at https://github.com/evalops/opencode/issues
# Tone and style
You should be concise, direct, and to the point.
diff --git a/packages/opencode/src/session/prompt/qwen.txt b/packages/opencode/src/session/prompt/qwen.txt
index a34fdb01a0..2fce3f9b17 100644
--- a/packages/opencode/src/session/prompt/qwen.txt
+++ b/packages/opencode/src/session/prompt/qwen.txt
@@ -6,7 +6,7 @@ IMPORTANT: You must NEVER generate or guess URLs for the user unless you are con
If the user asks for help or wants to give feedback inform them of the following:
- /help: Get help with using opencode
-- To give feedback, users should report the issue at https://github.com/sst/opencode/issues
+- To give feedback, users should report the issue at https://github.com/evalops/opencode/issues
When the user directly asks about opencode (eg 'can opencode do...', 'does opencode have...') or asks in second person (eg 'are you able...', 'can you do...'), first use the WebFetch tool to gather information to answer the question from opencode docs at https://opencode.ai
@@ -106,4 +106,3 @@ When referencing specific functions or pieces of code include the pattern `file_
user: Where are errors from the client handled?
assistant: Clients are marked as failed in the `connectToServer` function in src/services/process.ts:712.
-
diff --git a/packages/opencode/src/share/share.ts b/packages/opencode/src/share/share.ts
index 9df862d59a..285e8ab27f 100644
--- a/packages/opencode/src/share/share.ts
+++ b/packages/opencode/src/share/share.ts
@@ -9,6 +9,7 @@ export namespace Share {
let queue: Promise = Promise.resolve()
const pending = new Map()
+ const attempts = new Map()
export async function sync(key: string, content: any) {
const [root, ...splits] = key.split("/")
@@ -21,28 +22,56 @@ export namespace Share {
pending.set(key, content)
queue = queue
.then(async () => {
- const content = pending.get(key)
- if (content === undefined) return
- pending.delete(key)
-
- return fetch(`${URL}/share_sync`, {
- method: "POST",
- body: JSON.stringify({
- sessionID: sessionID,
- secret,
- key: key,
- content,
- }),
+ await flush(key, sessionID, secret)
+ })
+ .catch((error) => {
+ log.error("sync_failed", {
+ key: key,
+ error,
})
})
- .then((x) => {
- if (x) {
- log.info("synced", {
- key: key,
- status: x.status,
- })
- }
+ }
+
+ async function flush(key: string, sessionID: string, secret: string) {
+ while (true) {
+ const payload = pending.get(key)
+ if (payload === undefined) {
+ attempts.delete(key)
+ return
+ }
+
+ const attempt = (attempts.get(key) ?? 0) + 1
+ attempts.set(key, attempt)
+
+ const response = await fetch(`${URL}/share_sync`, {
+ method: "POST",
+ body: JSON.stringify({
+ sessionID: sessionID,
+ secret,
+ key: key,
+ content: payload,
+ }),
+ }).catch((error) => error as Error)
+
+ if (response instanceof Error || !response.ok) {
+ log.error("sync_retry", {
+ key: key,
+ attempt,
+ error: response instanceof Error ? response : response.status,
+ })
+ const delay = Math.min(30000, 1000 * 2 ** Math.min(attempt - 1, 5))
+ await new Promise((resolve) => setTimeout(resolve, delay))
+ continue
+ }
+
+ pending.delete(key)
+ attempts.delete(key)
+ log.info("synced", {
+ key: key,
+ status: response.status,
})
+ return
+ }
}
export function init() {
diff --git a/packages/opencode/src/snapshot/index.ts b/packages/opencode/src/snapshot/index.ts
index fb49ae739e..e2c855617f 100644
--- a/packages/opencode/src/snapshot/index.ts
+++ b/packages/opencode/src/snapshot/index.ts
@@ -40,7 +40,10 @@ export namespace Snapshot {
.nothrow()
log.info("initialized")
}
- await $`git --git-dir ${git} add .`.quiet().cwd(Instance.directory).nothrow()
+ await $`git -c core.excludesFile=/dev/null --git-dir ${git} add .`
+ .quiet()
+ .cwd(Instance.directory)
+ .nothrow()
const hash = await $`git --git-dir ${git} write-tree`.quiet().cwd(Instance.directory).nothrow().text()
log.info("tracking", { hash, cwd: Instance.directory, git })
return hash.trim()
@@ -54,7 +57,10 @@ export namespace Snapshot {
export async function patch(hash: string): Promise {
const git = gitdir()
- await $`git --git-dir ${git} add .`.quiet().cwd(Instance.directory).nothrow()
+ await $`git -c core.excludesFile=/dev/null --git-dir ${git} add .`
+ .quiet()
+ .cwd(Instance.directory)
+ .nothrow()
const result = await $`git --git-dir ${git} diff --name-only ${hash} -- .`.quiet().cwd(Instance.directory).nothrow()
// If git diff fails, return empty patch
@@ -64,6 +70,8 @@ export namespace Snapshot {
}
const files = result.text()
+ const directoryDisplay = Instance.directory
+ const directoryReal = await fs.realpath(directoryDisplay).catch(() => directoryDisplay)
return {
hash,
files: files
@@ -71,7 +79,8 @@ export namespace Snapshot {
.split("\n")
.map((x) => x.trim())
.filter(Boolean)
- .map((x) => path.join(Instance.worktree, x)),
+ .map((x) => path.join(Instance.worktree, x))
+ .map((abs) => (abs.startsWith(directoryReal) ? directoryDisplay + abs.slice(directoryReal.length) : abs)),
}
}
diff --git a/packages/opencode/src/storage/storage.ts b/packages/opencode/src/storage/storage.ts
index 546d123c6b..e3f3049e36 100644
--- a/packages/opencode/src/storage/storage.ts
+++ b/packages/opencode/src/storage/storage.ts
@@ -2,7 +2,6 @@ import { Log } from "../util/log"
import path from "path"
import fs from "fs/promises"
import { Global } from "../global"
-import { lazy } from "../util/lazy"
import { Lock } from "../util/lock"
import { $ } from "bun"
@@ -109,42 +108,77 @@ export namespace Storage {
},
]
- const state = lazy(async () => {
- const dir = path.join(Global.Path.data, "storage")
- const migration = await Bun.file(path.join(dir, "migration"))
- .json()
- .then((x) => parseInt(x))
- .catch(() => 0)
- for (let index = migration; index < MIGRATIONS.length; index++) {
- log.info("running migration", { index })
- const migration = MIGRATIONS[index]
- await migration(dir).catch((e) => {
- log.error("failed to run migration", { error: e, index })
- })
- await Bun.write(path.join(dir, "migration"), (index + 1).toString())
+ /**
+ * Get storage directory, respecting Instance context if available.
+ * Tests run in Instance.provide() will get isolated storage.
+ */
+ function getStorageDir(): string {
+ try {
+ const { Instance } = require("../project/instance")
+ const pending = Instance.pending
+ if (pending) {
+ return path.join(pending, ".opencode-storage")
+ }
+ const instanceDir = Instance.directory
+ if (instanceDir) {
+ return path.join(instanceDir, ".opencode-storage")
+ }
+ } catch {
+ // Not in Instance context, use global storage
}
- return {
- dir,
+ return path.join(Global.Path.data, "storage")
+ }
+
+ // Cache migrations per directory to avoid re-running
+ const migrationCache = new Map>()
+
+ async function ensureMigrations(dir: string): Promise {
+ if (migrationCache.has(dir)) {
+ return migrationCache.get(dir)!
}
- })
+
+ const promise = (async () => {
+ const migration = await Bun.file(path.join(dir, "migration"))
+ .json()
+ .then((x) => parseInt(x))
+ .catch(() => 0)
+ for (let index = migration; index < MIGRATIONS.length; index++) {
+ log.info("running migration", { index, dir })
+ const migration = MIGRATIONS[index]
+ await migration(dir).catch((e) => {
+ log.error("failed to run migration", { error: e, index, dir })
+ })
+ await Bun.write(path.join(dir, "migration"), (index + 1).toString())
+ }
+ })()
+
+ migrationCache.set(dir, promise)
+ return promise
+ }
+
+ async function getDir(): Promise {
+ const dir = getStorageDir()
+ await ensureMigrations(dir)
+ return dir
+ }
export async function remove(key: string[]) {
- const dir = await state().then((x) => x.dir)
+ const dir = await getDir()
const target = path.join(dir, ...key) + ".json"
await fs.unlink(target).catch(() => {})
}
export async function read(key: string[]) {
- const dir = await state().then((x) => x.dir)
+ const dir = await getDir()
const target = path.join(dir, ...key) + ".json"
using _ = await Lock.read(target)
return Bun.file(target).json() as Promise
}
export async function update(key: string[], fn: (draft: T) => void) {
- const dir = await state().then((x) => x.dir)
+ const dir = await getDir()
const target = path.join(dir, ...key) + ".json"
- using _ = await Lock.write("storage")
+ using _ = await Lock.write(target)
const content = await Bun.file(target).json()
fn(content)
await Bun.write(target, JSON.stringify(content, null, 2))
@@ -152,15 +186,16 @@ export namespace Storage {
}
export async function write(key: string[], content: T) {
- const dir = await state().then((x) => x.dir)
+ const dir = await getDir()
const target = path.join(dir, ...key) + ".json"
- using _ = await Lock.write("storage")
+ using _ = await Lock.write(target)
+ await fs.mkdir(path.dirname(target), { recursive: true }).catch(() => {})
await Bun.write(target, JSON.stringify(content, null, 2))
}
const glob = new Bun.Glob("**/*")
export async function list(prefix: string[]) {
- const dir = await state().then((x) => x.dir)
+ const dir = await getDir()
try {
const result = await Array.fromAsync(
glob.scan({
diff --git a/packages/opencode/src/tool/bash.ts b/packages/opencode/src/tool/bash.ts
index ddf8227e9e..7a4d8acfa6 100644
--- a/packages/opencode/src/tool/bash.ts
+++ b/packages/opencode/src/tool/bash.ts
@@ -1,16 +1,16 @@
import z from "zod/v4"
-import { exec } from "child_process"
import { Tool } from "./tool"
import DESCRIPTION from "./bash.txt"
import { Permission } from "../permission"
-import { Filesystem } from "../util/filesystem"
import { lazy } from "../util/lazy"
import { Log } from "../util/log"
import { Wildcard } from "../util/wildcard"
import { $ } from "bun"
import { Instance } from "../project/instance"
import { Agent } from "../agent/agent"
+import { measure } from "./telemetry"
+import { guard } from "./workspace"
const MAX_OUTPUT_LENGTH = 30_000
const DEFAULT_TIMEOUT = 1 * 60 * 1000
@@ -43,24 +43,39 @@ const parser = lazy(async () => {
}
})
-export const BashTool = Tool.define("bash", {
+const parameters = z.object({
+ command: z.string().describe("The command to execute"),
+ timeout: z.number().describe("Optional timeout in milliseconds").optional(),
+ description: z
+ .string()
+ .describe(
+ "Clear, concise description of what this command does in 5-10 words. Examples:\nInput: ls\nOutput: Lists files in current directory\n\nInput: git status\nOutput: Shows working tree status\n\nInput: npm install\nOutput: Installs package dependencies\n\nInput: mkdir foo\nOutput: Creates directory 'foo'",
+ ),
+})
+
+type BashMetadata = {
+ output: string
+ exit?: number
+ description: string
+}
+
+export const BashTool = Tool.define("bash", {
description: DESCRIPTION,
- parameters: z.object({
- command: z.string().describe("The command to execute"),
- timeout: z.number().describe("Optional timeout in milliseconds").optional(),
- description: z
- .string()
- .describe(
- "Clear, concise description of what this command does in 5-10 words. Examples:\nInput: ls\nOutput: Lists files in current directory\n\nInput: git status\nOutput: Shows working tree status\n\nInput: npm install\nOutput: Installs package dependencies\n\nInput: mkdir foo\nOutput: Creates directory 'foo'",
- ),
- }),
+ parameters,
async execute(params, ctx) {
- const timeout = Math.min(params.timeout ?? DEFAULT_TIMEOUT, MAX_TIMEOUT)
- const tree = await parser().then((p) => p.parse(params.command))
- const permissions = await Agent.get(ctx.agent).then((x) => x.permission.bash)
-
- const askPatterns = new Set()
- for (const node of tree.rootNode.descendantsOfType("command")) {
+ const extra = { description: params.description }
+ return measure({
+ id: "bash",
+ ctx,
+ params,
+ extra,
+ async run() {
+ const timeout = Math.min(params.timeout ?? DEFAULT_TIMEOUT, MAX_TIMEOUT)
+ const tree = await parser().then((p) => p.parse(params.command))
+ const permissions = await Agent.get(ctx.agent).then((x) => x.permission.bash)
+
+ const askPatterns = new Set()
+ for (const node of tree.rootNode.descendantsOfType("command")) {
const command = []
for (let i = 0; i < node.childCount; i++) {
const child = node.child(i)
@@ -87,11 +102,10 @@ export const BashTool = Tool.define("bash", {
.text()
.then((x) => x.trim())
log.info("resolved path", { arg, resolved })
- if (resolved && !Filesystem.contains(Instance.directory, resolved)) {
- throw new Error(
- `This command references paths outside of ${Instance.directory} so it is not allowed to be executed.`,
- )
- }
+ if (resolved)
+ guard(resolved, {
+ message: `This command references paths outside of ${Instance.directory} so it is not allowed to be executed.`,
+ })
}
}
@@ -130,7 +144,7 @@ export const BashTool = Tool.define("bash", {
}
}
- if (askPatterns.size > 0) {
+ if (askPatterns.size > 0) {
const patterns = Array.from(askPatterns)
await Permission.ask({
type: "bash",
@@ -146,69 +160,74 @@ export const BashTool = Tool.define("bash", {
})
}
- const process = exec(params.command, {
- cwd: Instance.directory,
- signal: ctx.abort,
- timeout,
- })
-
- let output = ""
-
- // Initialize metadata with empty output
- ctx.metadata({
- metadata: {
- output: "",
- description: params.description,
- },
- })
-
- process.stdout?.on("data", (chunk) => {
- output += chunk.toString()
- ctx.metadata({
- metadata: {
- output: output,
- description: params.description,
- },
- })
- })
-
- process.stderr?.on("data", (chunk) => {
- output += chunk.toString()
- ctx.metadata({
- metadata: {
- output: output,
- description: params.description,
- },
- })
- })
+ const controller = new AbortController()
+ const timer = setTimeout(() => controller.abort(), timeout)
+ const signal = AbortSignal.any([ctx.abort, controller.signal])
+ const shell = process.env["SHELL"] || "/bin/sh"
+ const proc = Bun.spawn([shell, "-lc", params.command], {
+ cwd: Instance.directory,
+ stdout: "pipe",
+ stderr: "pipe",
+ signal,
+ })
+
+ const state = { output: "" }
+ const decoder = () => new TextDecoder()
+ const pump = async (stream: ReadableStream | undefined) => {
+ if (!stream) return
+ const textDecoder = decoder()
+ await stream.pipeTo(
+ new WritableStream({
+ write(chunk) {
+ const text = textDecoder.decode(chunk, { stream: true })
+ if (!text) return
+ state.output += text
+ ctx.metadata({
+ metadata: {
+ output: state.output,
+ description: params.description,
+ },
+ })
+ },
+ }),
+ )
+ }
- await new Promise((resolve) => {
- process.on("close", () => {
- resolve()
- })
- })
+ ctx.metadata({
+ metadata: {
+ output: "",
+ description: params.description,
+ },
+ })
+
+ await Promise.all([pump(proc.stdout), pump(proc.stderr)])
+ const exit = await proc.exited
+ clearTimeout(timer)
+
+ ctx.metadata({
+ metadata: {
+ output: state.output,
+ exit,
+ description: params.description,
+ },
+ })
+
+ let finalOutput = state.output
+ if (finalOutput.length > MAX_OUTPUT_LENGTH) {
+ finalOutput = finalOutput.slice(0, MAX_OUTPUT_LENGTH)
+ finalOutput += "\n\n(Output was truncated due to length limit)"
+ }
- ctx.metadata({
- metadata: {
- output: output,
- exit: process.exitCode,
- description: params.description,
+ return {
+ title: params.command,
+ metadata: {
+ output: finalOutput,
+ exit,
+ description: params.description,
+ },
+ output: finalOutput,
+ }
},
})
-
- if (output.length > MAX_OUTPUT_LENGTH) {
- output = output.slice(0, MAX_OUTPUT_LENGTH)
- output += "\n\n(Output was truncated due to length limit)"
- }
-
- return {
- title: params.command,
- metadata: {
- output,
- exit: process.exitCode,
- description: params.description,
- },
- output,
- }
},
})
diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts
index 579f9f09d8..a0e7c475ae 100644
--- a/packages/opencode/src/tool/edit.ts
+++ b/packages/opencode/src/tool/edit.ts
@@ -13,9 +13,9 @@ import DESCRIPTION from "./edit.txt"
import { File } from "../file"
import { Bus } from "../bus"
import { FileTime } from "../file/time"
-import { Filesystem } from "../util/filesystem"
import { Instance } from "../project/instance"
import { Agent } from "../agent/agent"
+import { guard } from "./workspace"
export const EditTool = Tool.define("edit", {
description: DESCRIPTION,
@@ -34,10 +34,7 @@ export const EditTool = Tool.define("edit", {
throw new Error("oldString and newString must be different")
}
- const filePath = path.isAbsolute(params.filePath) ? params.filePath : path.join(Instance.directory, params.filePath)
- if (!Filesystem.contains(Instance.directory, filePath)) {
- throw new Error(`File ${filePath} is not in the current working directory`)
- }
+ const filePath = guard(params.filePath)
const agent = await Agent.get(ctx.agent)
let diff = ""
diff --git a/packages/opencode/src/tool/exitspecmode.ts b/packages/opencode/src/tool/exitspecmode.ts
new file mode 100644
index 0000000000..ddc402d9ea
--- /dev/null
+++ b/packages/opencode/src/tool/exitspecmode.ts
@@ -0,0 +1,81 @@
+import z from "zod/v4"
+import { Tool } from "./tool"
+import DESCRIPTION from "./exitspecmode.txt"
+import { getSpecState } from "./specmode"
+
+export const ExitSpecModeTool = Tool.define("exitspecmode", {
+ description: DESCRIPTION,
+ parameters: z.object({
+ plan: z.string().describe("The markdown-formatted plan you came up with"),
+ title: z.string().optional().describe("Optional title for the plan"),
+ include_context: z
+ .boolean()
+ .optional()
+ .describe("Include requirements and notes from SpecMode (default true)"),
+ }),
+ async execute(params, ctx) {
+ const specSessions = getSpecState()
+ const session = specSessions[ctx.sessionID]
+
+ // Check if in spec mode
+ if (!session?.active) {
+ // Allow exiting even if not in spec mode, but warn
+ return {
+ title: params.title || "Implementation Plan",
+ output:
+ "⚠️ Warning: Not currently in spec mode\n\n" + params.plan,
+ metadata: {
+ was_in_spec_mode: false,
+ requirements_count: 0,
+ notes_count: 0,
+ duration_seconds: 0,
+ },
+ }
+ }
+
+ const includeContext = params.include_context !== false
+
+ // Build output with optional context
+ let output = ""
+
+ if (includeContext && (session.requirements.length > 0 || session.notes.length > 0)) {
+ output += "## Spec Context\n\n"
+
+ if (session.requirements.length > 0) {
+ output += "### Requirements\n"
+ session.requirements.forEach((req, i) => {
+ output += `${i + 1}. ${req}\n`
+ })
+ output += "\n"
+ }
+
+ if (session.notes.length > 0) {
+ output += "### Planning Notes\n"
+ session.notes.forEach((note, i) => {
+ output += `${i + 1}. ${note}\n`
+ })
+ output += "\n"
+ }
+
+ output += "---\n\n"
+ }
+
+ output += params.plan
+
+ // Deactivate spec mode
+ session.active = false
+
+ const title = params.title || "Implementation Plan"
+
+ return {
+ title,
+ output,
+ metadata: {
+ was_in_spec_mode: true,
+ requirements_count: session.requirements.length,
+ notes_count: session.notes.length,
+ duration_seconds: Math.floor((Date.now() - session.startedAt) / 1000),
+ },
+ }
+ },
+})
diff --git a/packages/opencode/src/tool/exitspecmode.txt b/packages/opencode/src/tool/exitspecmode.txt
new file mode 100644
index 0000000000..bdaef68acc
--- /dev/null
+++ b/packages/opencode/src/tool/exitspecmode.txt
@@ -0,0 +1,24 @@
+Exit specification/planning mode and present your final implementation plan.
+
+Integration with SpecMode:
+- If you used SpecMode to collect requirements and notes, this tool will automatically include them in the output
+- The requirements and notes are prepended to your plan as context
+- Set `include_context: false` if you don't want the requirements/notes included
+- This tool deactivates the spec session after use
+
+Usage:
+- `plan` (required) - Your markdown-formatted implementation plan
+- `title` (optional) - Title for the plan (defaults to "Implementation Plan")
+- `include_context` (optional) - Include SpecMode requirements/notes (defaults to true)
+
+IMPORTANT: Only use this tool when the task requires planning the implementation steps of a task that requires writing code. For research tasks where you're gathering information, searching files, reading files or in general trying to understand the codebase - do NOT use this tool.
+
+Examples:
+1. Initial task: "Search for and understand the implementation of vim mode in the codebase" - Do not use this tool because you are not planning the implementation steps of a task.
+2. Initial task: "Help me implement yank mode for vim" - Use this tool after you have finished planning the implementation steps of the task.
+
+Workflow:
+1. Optionally use SpecMode to collect requirements and notes
+2. Create your implementation plan
+3. Use this tool to present the plan and exit spec mode
+4. The output will include SpecMode context (if collected) followed by your plan
diff --git a/packages/opencode/src/tool/fetchurl.ts b/packages/opencode/src/tool/fetchurl.ts
new file mode 100644
index 0000000000..94ddcd9de4
--- /dev/null
+++ b/packages/opencode/src/tool/fetchurl.ts
@@ -0,0 +1,454 @@
+import z from "zod/v4"
+import { Tool } from "./tool"
+import TurndownService from "turndown"
+import { Octokit } from "@octokit/rest"
+import { parseHTML } from "linkedom"
+import DESCRIPTION from "./fetchurl.txt"
+import { Config } from "../config/config"
+import { Permission } from "../permission"
+import { measure } from "./telemetry"
+
+const MAX_RESPONSE_SIZE = 10 * 1024 * 1024 // 10MB
+const DEFAULT_TIMEOUT = 30 * 1000 // 30 seconds
+const MAX_TIMEOUT = 120 * 1000 // 2 minutes
+const MAX_REDIRECTS = 5
+
+// Private IP ranges to block
+const PRIVATE_IP_RANGES = [
+ /^127\./, // 127.0.0.0/8
+ /^10\./, // 10.0.0.0/8
+ /^172\.(1[6-9]|2[0-9]|3[0-1])\./, // 172.16.0.0/12
+ /^192\.168\./, // 192.168.0.0/16
+ /^localhost$/i,
+ /^::1$/, // IPv6 localhost
+ /^fe80::/i, // IPv6 link-local
+]
+
+const schema = z.object({
+ url: z.string().describe("The URL to fetch content from"),
+ format: z
+ .enum(["markdown", "text", "html", "json", "auto"])
+ .optional()
+ .describe("Output format (auto-detected if not specified)"),
+ integration: z
+ .enum(["google_docs", "notion", "linear", "github", "gitlab", "jira", "pagerduty", "slack", "sentry", "generic"])
+ .optional()
+ .describe("Integration type (auto-detected if not specified)"),
+ auth_type: z
+ .enum(["bearer", "api_key", "header", "query", "none"])
+ .optional()
+ .describe("Authentication type"),
+ auth_token: z.string().optional().describe("Authentication token/API key"),
+ auth_header_name: z.string().optional().describe("Custom header name for auth (if auth_type=header)"),
+ auth_query_param: z.string().optional().describe("Query parameter name for auth (if auth_type=query)"),
+ timeout: z.number().optional().describe("Optional timeout in seconds (max 120)"),
+ follow_redirects: z.boolean().optional().describe("Follow HTTP redirects (default true, max 5)"),
+})
+
+type FetchArgs = z.infer
+type FetchMeta = {
+ integration: string
+ api_used: boolean
+ content_type: string
+ size: number
+ redirects?: number
+ final_url?: string
+ status?: number
+}
+
+export const FetchUrlTool = Tool.define("fetchurl", {
+ description: DESCRIPTION,
+ parameters: schema,
+ async execute(params, ctx) {
+ const telemetryExtra: Record = {}
+ return measure({
+ id: "fetchurl",
+ ctx,
+ params,
+ extra: telemetryExtra,
+ async run() {
+ // Validate URL and check for private IPs
+ if (!params.url.startsWith("http://") && !params.url.startsWith("https://")) {
+ throw new Error("URL must start with http:// or https://")
+ }
+
+ // Extract hostname and check against private IP patterns
+ const hostname = new URL(params.url).hostname
+ for (const pattern of PRIVATE_IP_RANGES) {
+ if (pattern.test(hostname)) {
+ throw new Error("Access to localhost and private IP addresses is not allowed")
+ }
+ }
+
+ const cfg = await Config.get()
+ if (cfg.permission?.fetchurl === "ask")
+ await Permission.ask({
+ type: "fetchurl",
+ sessionID: ctx.sessionID,
+ messageID: ctx.messageID,
+ callID: ctx.callID,
+ title: "Fetch content from: " + params.url,
+ metadata: {
+ url: params.url,
+ integration: params.integration,
+ },
+ })
+
+ const timeout = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT)
+
+ const integration = params.integration || detectIntegration(params.url)
+ telemetryExtra["integration"] = integration
+
+ if (integration === "github" && canUseGitHubAPI(params.url, params.auth_token)) {
+ const content = await fetchGitHubContent(params.url, params.auth_token, params.format)
+ telemetryExtra["api_used"] = true
+ telemetryExtra["final_url"] = params.url
+ const metadata = {
+ integration: "github",
+ api_used: true,
+ content_type: "api/json",
+ size: content.length,
+ }
+ return {
+ title: `${params.url} (github-api)`,
+ output: content,
+ metadata,
+ }
+ }
+
+ const result = await fetchHTTP(params, ctx, integration, timeout, telemetryExtra)
+ return result
+ },
+ })
+ },
+})
+
+function detectIntegration(url: string): string {
+ const hostname = new URL(url).hostname.toLowerCase()
+
+ if (hostname.includes("docs.google.com")) return "google_docs"
+ if (hostname.includes("notion.so") || hostname.includes("notion.site")) return "notion"
+ if (hostname.includes("linear.app")) return "linear"
+ if (hostname.includes("github.com")) return "github"
+ if (hostname.includes("gitlab.com")) return "gitlab"
+ if (hostname.includes("atlassian.net") || hostname.includes("jira.")) return "jira"
+ if (hostname.includes("pagerduty.com")) return "pagerduty"
+ if (hostname.includes("slack.com")) return "slack"
+ if (hostname.includes("sentry.io")) return "sentry"
+
+ return "generic"
+}
+
+function canUseGitHubAPI(url: string, authToken?: string): boolean {
+ // Check if URL is a GitHub file/repo URL and we have an auth token
+ return url.includes("github.com") && (!!authToken || !!process.env["GITHUB_TOKEN"])
+}
+
+async function fetchGitHubContent(url: string, authToken?: string, format?: string): Promise {
+ const token = authToken || process.env["GITHUB_TOKEN"]
+ const octokit = new Octokit({ auth: token })
+
+ // Parse GitHub URL
+ const match = url.match(/github\.com\/([^\/]+)\/([^\/]+)(?:\/(?:blob|tree)\/([^\/]+)\/(.+))?/)
+ if (!match) {
+ throw new Error("Invalid GitHub URL format")
+ }
+
+ const [, owner, repo, ref, path] = match
+
+ // If no path, fetch README
+ if (!path) {
+ const { data } = await octokit.repos.getReadme({ owner, repo, ref })
+ const content = Buffer.from(data.content, "base64").toString()
+ if (format === "html") return content
+ if (format === "text") return stripMarkdown(content)
+ return content // markdown by default
+ }
+
+ // Fetch file content
+ const { data } = await octokit.repos.getContent({ owner, repo, path, ref })
+
+ if (Array.isArray(data)) {
+ // Directory listing
+ let markdown = `# Directory: ${path}\n\n`
+ for (const item of data) {
+ markdown += `- [${item.type === "dir" ? "📁" : "📄"} ${item.name}](${item.html_url})\n`
+ }
+ return markdown
+ }
+
+ if ("content" in data) {
+ const content = Buffer.from(data.content, "base64").toString()
+ const ext = path.split(".").pop()?.toLowerCase()
+
+ if (format === "text") return content
+ if (format === "html") return `${escapeHTML(content)}
`
+
+ // Return with syntax highlighting info
+ return `\`\`\`${ext}\n${content}\n\`\`\``
+ }
+
+ throw new Error("Could not fetch GitHub content")
+}
+
+async function fetchHTTP(
+ params: FetchArgs,
+ ctx: Tool.Context,
+ integration: string,
+ timeout: number,
+ telemetryExtra: Record,
+): Promise<{ title: string; output: string; metadata: FetchMeta }> {
+ const state = { url: params.url, redirects: 0 }
+ const chain = [params.url]
+ const follow = params.follow_redirects !== false
+
+ const headers: Record = {
+ "User-Agent":
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+ Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+ }
+
+ if (params.auth_token) {
+ const authType = params.auth_type || "bearer"
+ if (authType === "bearer") headers["Authorization"] = `Bearer ${params.auth_token}`
+ if (authType === "api_key") headers["X-API-Key"] = params.auth_token
+ if (authType === "header" && params.auth_header_name) headers[params.auth_header_name] = params.auth_token
+ if (authType === "query" && params.auth_query_param) {
+ const urlObj = new URL(state.url)
+ urlObj.searchParams.set(params.auth_query_param, params.auth_token)
+ state.url = urlObj.toString()
+ }
+ }
+
+ while (true) {
+ const controller = new AbortController()
+ const timer = setTimeout(() => controller.abort(), timeout)
+
+ const response = await fetch(state.url, {
+ signal: AbortSignal.any([controller.signal, ctx.abort]),
+ headers,
+ redirect: "manual",
+ })
+
+ clearTimeout(timer)
+
+ const isRedirect =
+ response.status === 301 || response.status === 302 || response.status === 307 || response.status === 308
+ if (follow && isRedirect) {
+ const location = response.headers.get("location")
+ if (!location) throw new Error(`Redirect without location header`)
+
+ state.redirects += 1
+ if (state.redirects > MAX_REDIRECTS) throw new Error(`Too many redirects (max ${MAX_REDIRECTS})`)
+
+ state.url = new URL(location, state.url).toString()
+ chain.push(state.url)
+ const newHostname = new URL(state.url).hostname
+ for (const pattern of PRIVATE_IP_RANGES) {
+ if (pattern.test(newHostname)) throw new Error("Redirect to localhost/private IP is not allowed")
+ }
+
+ continue
+ }
+
+ if (!response.ok) throw new Error(`Request failed with status code: ${response.status}`)
+
+ const contentLength = response.headers.get("content-length")
+ if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
+ throw new Error(`Response too large (exceeds ${MAX_RESPONSE_SIZE / 1024 / 1024}MB limit)`)
+ }
+
+ const arrayBuffer = await response.arrayBuffer()
+ if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+ throw new Error(`Response too large (exceeds ${MAX_RESPONSE_SIZE / 1024 / 1024}MB limit)`)
+ }
+
+ const content = new TextDecoder().decode(arrayBuffer)
+ const contentType = response.headers.get("content-type") || ""
+ const output = await processContent(content, contentType, integration, state.url, params.format)
+ telemetryExtra["final_url"] = state.url
+ telemetryExtra["redirects"] = state.redirects
+ telemetryExtra["redirect_chain"] = chain
+ telemetryExtra["content_type"] = contentType
+ telemetryExtra["status"] = response.status
+
+ return {
+ title: `${state.url} (${integration})`,
+ output,
+ metadata: {
+ integration,
+ api_used: false,
+ content_type: contentType,
+ size: arrayBuffer.byteLength,
+ redirects: state.redirects,
+ final_url: state.url,
+ status: response.status,
+ },
+ }
+ }
+}
+
+async function processContent(
+ content: string,
+ contentType: string,
+ integration: string,
+ url: string,
+ formatPreference?: FetchArgs["format"],
+): Promise {
+ // Handle JSON responses
+ if (contentType.includes("application/json")) {
+ if (formatPreference === "json" || formatPreference === "text") {
+ return content
+ }
+ try {
+ const json = JSON.parse(content)
+ return formatJSONAsMarkdown(json, integration)
+ } catch {
+ return content
+ }
+ }
+
+ // Handle HTML content
+ if (contentType.includes("text/html")) {
+ if (formatPreference === "html") {
+ return content
+ }
+ if (formatPreference === "text") {
+ return extractTextFromHTML(content)
+ }
+ // Default to markdown
+ return convertHTMLToMarkdown(content, integration, url)
+ }
+
+ // Plain text or other
+ return content
+}
+
+function formatJSONAsMarkdown(data: unknown, integration: string): string {
+ const heading = `# ${integration.toUpperCase()} Content`
+ const parts = [heading, ""]
+ const record = toRecord(data)
+
+ if (integration === "github" && record) {
+ const name = record["name"]
+ if (typeof name === "string" && name) parts.push(`## ${name}`, "")
+ const description = record["description"]
+ if (typeof description === "string" && description) parts.push(description, "")
+ const content = record["content"]
+ if (typeof content === "string" && content) {
+ const decoded = Buffer.from(content, "base64").toString()
+ parts.push("```", decoded, "```", "")
+ }
+ return parts.join("\n").trimEnd()
+ }
+
+ if (integration === "linear" && record) {
+ const title = record["title"]
+ if (typeof title === "string" && title) parts.push(`## ${title}`, "")
+ const description = record["description"]
+ if (typeof description === "string" && description) parts.push(description, "")
+ const state = record["state"]
+ if (typeof state === "string" && state) parts.push(`**State:** ${state}`, "")
+ return parts.join("\n").trimEnd()
+ }
+
+ if (integration === "jira" && record) {
+ const fields = toRecord(record["fields"])
+ if (fields) {
+ const summary = fields["summary"]
+ if (typeof summary === "string" && summary) parts.push(`## ${summary}`, "")
+ const description = fields["description"]
+ if (typeof description === "string" && description) parts.push(description, "")
+ const status = toRecord(fields["status"])
+ const name = status ? status["name"] : undefined
+ if (typeof name === "string" && name) parts.push(`**Status:** ${name}`, "")
+ }
+ return parts.join("\n").trimEnd()
+ }
+
+ parts.push("```json", JSON.stringify(data, null, 2), "```", "")
+ return parts.join("\n").trimEnd()
+}
+
+function toRecord(value: unknown): Record | null {
+ if (value && typeof value === "object") return value as Record
+ return null
+}
+
+function convertHTMLToMarkdown(html: string, integration: string, url: string): string {
+ // First extract relevant content based on integration
+ const extracted = extractIntegrationContent(html, integration)
+
+ const turndownService = new TurndownService({
+ headingStyle: "atx",
+ hr: "---",
+ bulletListMarker: "-",
+ codeBlockStyle: "fenced",
+ emDelimiter: "*",
+ })
+
+ turndownService.remove(["script", "style", "meta", "link", "nav", "footer", "header"])
+
+ let markdown = turndownService.turndown(extracted)
+ markdown = `# Content from ${url}\n\n${markdown}`
+
+ return markdown.trim()
+}
+
+function extractIntegrationContent(html: string, integration: string): string {
+ try {
+ const { document } = parseHTML(html)
+
+ switch (integration) {
+ case "google_docs":
+ const docsContent = document.querySelector(".kix-appview-editor")
+ return docsContent?.innerHTML || html
+ case "notion":
+ const notionContent = document.querySelector(".notion-page-content")
+ return notionContent?.innerHTML || html
+ case "github":
+ const ghContent =
+ document.querySelector(".markdown-body") || document.querySelector(".highlight")
+ return ghContent?.innerHTML || html
+ default:
+ // Try to find main content
+ const main = document.querySelector("main") || document.querySelector("article") || document.querySelector("body")
+ return main?.innerHTML || html
+ }
+ } catch {
+ return html
+ }
+}
+
+function extractTextFromHTML(html: string): string {
+ try {
+ const { document } = parseHTML(html)
+ return document.body?.textContent?.trim() || html
+ } catch {
+ return html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim()
+ }
+}
+
+function stripMarkdown(content: string): string {
+ return content
+ .replace(/#+\s/g, "")
+ .replace(/\*\*([^*]+)\*\*/g, "$1")
+ .replace(/\*([^*]+)\*/g, "$1")
+ .replace(/`([^`]+)`/g, "$1")
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
+ .trim()
+}
+
+function escapeHTML(text: string): string {
+ return text.replace(/[&<>"']/g, (char) => {
+ const escape: Record = {
+ "&": "&",
+ "<": "<",
+ ">": ">",
+ '"': """,
+ "'": "'",
+ }
+ return escape[char] || char
+ })
+}
diff --git a/packages/opencode/src/tool/fetchurl.txt b/packages/opencode/src/tool/fetchurl.txt
new file mode 100644
index 0000000000..67f8425ec9
--- /dev/null
+++ b/packages/opencode/src/tool/fetchurl.txt
@@ -0,0 +1,13 @@
+URL content fetching and scraping tool with integration support
+
+Features:
+- Generic webpage scraping with markdown conversion
+- Integration support for: Google Docs, Notion, Linear, GitHub, GitLab, Jira, PagerDuty, Slack, Sentry
+- Returns content in markdown format
+- Security: Blocks localhost and private IP addresses
+- Configurable timeout and size limits
+
+Usage notes:
+- Automatically detects and handles platform-specific content formats
+- Supports both public URLs and authenticated endpoints (when credentials are provided)
+- Returns structured markdown for better readability
diff --git a/packages/opencode/src/tool/history.ts b/packages/opencode/src/tool/history.ts
new file mode 100644
index 0000000000..87e6ec482d
--- /dev/null
+++ b/packages/opencode/src/tool/history.ts
@@ -0,0 +1,50 @@
+import { Storage } from "../storage/storage"
+import type { TelemetryEvent } from "./telemetry-event"
+
+const KEY = ["telemetry", "tools"]
+const MAX_EVENTS = 200
+
+type TelemetrySummary = {
+ version: 1
+ tools: Record
+ events: TelemetryEvent[]
+}
+
+async function ensure(): Promise {
+ try {
+ return await Storage.read(KEY)
+ } catch {
+ const fresh: TelemetrySummary = { version: 1, tools: {}, events: [] }
+ await Storage.write(KEY, fresh)
+ return fresh
+ }
+}
+
+async function write(summary: TelemetrySummary) {
+ await Storage.write(KEY, summary)
+}
+
+export namespace ToolHistory {
+ export async function record(event: TelemetryEvent) {
+ const summary = await ensure()
+ const entry = (summary.tools[event.id] ??= { runs: 0, errors: 0, totalDuration: 0 })
+ entry.runs += 1
+ entry.totalDuration += event.duration
+ if (event.status === "error") entry.errors += 1
+
+ summary.events.push(event)
+ if (summary.events.length > MAX_EVENTS) {
+ summary.events.splice(0, summary.events.length - MAX_EVENTS)
+ }
+ await write(summary)
+ }
+
+ export async function read(): Promise {
+ return ensure()
+ }
+
+ export async function clear() {
+ const fresh: TelemetrySummary = { version: 1, tools: {}, events: [] }
+ await write(fresh)
+ }
+}
diff --git a/packages/opencode/src/tool/ls.ts b/packages/opencode/src/tool/ls.ts
index b80f668a5b..b815fd766a 100644
--- a/packages/opencode/src/tool/ls.ts
+++ b/packages/opencode/src/tool/ls.ts
@@ -4,6 +4,7 @@ import * as path from "path"
import DESCRIPTION from "./ls.txt"
import { Instance } from "../project/instance"
import { Ripgrep } from "../file/ripgrep"
+import { measure } from "./telemetry"
export const IGNORE_PATTERNS = [
"node_modules/",
@@ -40,71 +41,87 @@ export const ListTool = Tool.define("list", {
path: z.string().describe("The absolute path to the directory to list (must be absolute, not relative)").optional(),
ignore: z.array(z.string()).describe("List of glob patterns to ignore").optional(),
}),
- async execute(params) {
+ async execute(params, ctx) {
const searchPath = path.resolve(Instance.directory, params.path || ".")
-
- const ignoreGlobs = IGNORE_PATTERNS.map((p) => `!${p}*`).concat(params.ignore?.map((p) => `!${p}`) || [])
- const files = []
- for await (const file of Ripgrep.files({ cwd: searchPath, glob: ignoreGlobs })) {
- files.push(file)
- if (files.length >= LIMIT) break
- }
-
- // Build directory structure
- const dirs = new Set()
- const filesByDir = new Map()
-
- for (const file of files) {
- const dir = path.dirname(file)
- const parts = dir === "." ? [] : dir.split("/")
-
- // Add all parent directories
- for (let i = 0; i <= parts.length; i++) {
- const dirPath = i === 0 ? "." : parts.slice(0, i).join("/")
- dirs.add(dirPath)
- }
-
- // Add file to its directory
- if (!filesByDir.has(dir)) filesByDir.set(dir, [])
- filesByDir.get(dir)!.push(path.basename(file))
- }
-
- function renderDir(dirPath: string, depth: number): string {
- const indent = " ".repeat(depth)
- let output = ""
-
- if (depth > 0) {
- output += `${indent}${path.basename(dirPath)}/\n`
- }
-
- const childIndent = " ".repeat(depth + 1)
- const children = Array.from(dirs)
- .filter((d) => path.dirname(d) === dirPath && d !== dirPath)
- .sort()
-
- // Render subdirectories first
- for (const child of children) {
- output += renderDir(child, depth + 1)
- }
-
- // Render files
- const files = filesByDir.get(dirPath) || []
- for (const file of files.sort()) {
- output += `${childIndent}${file}\n`
- }
-
- return output
- }
-
- const output = `${searchPath}/\n` + renderDir(".", 0)
-
- return {
- title: path.relative(Instance.worktree, searchPath),
- metadata: {
- count: files.length,
- truncated: files.length >= LIMIT,
+ const relativePath = path.relative(Instance.worktree, searchPath)
+
+ return measure({
+ id: "list",
+ ctx,
+ params,
+ captureInput: () => ({
+ path: relativePath,
+ ignore: params.ignore,
+ }),
+ captureOutput: (result) => ({
+ count: result.metadata ? (result.metadata as Record 
- 
