{
  "schema_version": "2026-05-28.agent-improvement-actions.v1",
  "name": "TokRepo Agent Improvement Actions Eval",
  "canonical_url": "https://tokrepo.com/evals/agent-improvement-actions.json",
  "updated_at": "2026-05-28",
  "generated_by": "node scripts/run_agent_improvement_actions.mjs --json",
  "purpose": "Public deterministic contract for turning TokRepo agent quality feedback into actionable, privacy-safe repair recommendations that agents can execute before claiming a long-term loop improved.",
  "improvement_action_contract": {
    "public_feedback_api": "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel",
    "event_ingestion_api": "https://api.tokrepo.com/api/v1/tokenboard/agent/events",
    "required_live_window_days": 14,
    "required_live_event_keys": [
      "mcp_search",
      "mcp_detail",
      "capability_resolve",
      "verify_asset",
      "audit_asset",
      "install_plan",
      "rollback_plan",
      "agent_handoff",
      "handoff_plan",
      "harvest_plan"
    ],
    "optional_live_event_keys": [
      "harvest_publish"
    ],
    "action_boundary": "An improvement action is not just a metric. It must name a trigger, required events, a repair command, an expected artifact, an acceptance oracle, and privacy guards.",
    "completion_boundary": "TokRepo must not claim self-improvement from search, install, or handoff counts alone. A passing action links quality feedback to an explicit repair path and verifies it with public eval evidence.",
    "privacy_boundary": "Only anonymous aggregate counters, public schema references, action ids, deterministic coverage metadata, and hashed identifiers may be published. The contract must not expose query text, task text, file contents, raw diffs, local paths, credentials, cookies, share tokens, or private harvest payloads.",
    "live_check_command": "WEB_BASE=https://tokrepo.com API_BASE=https://api.tokrepo.com node scripts/run_agent_improvement_actions.mjs --check-live"
  },
  "metrics": {
    "actions_total": 3,
    "actions_passed": 3,
    "improvement_action_coverage_pct": 100,
    "live_funnel_dependency_coverage_pct": 100,
    "actionability_coverage_pct": 100,
    "privacy_guard_coverage_pct": 100
  },
  "improvement_actions": [
    {
      "action_id": "repair_relevance_resolution_gap",
      "trigger": "Search volume exists but useful detail reads or capability resolution are too low for the requested task category.",
      "required_events": [
        "mcp_search",
        "mcp_detail",
        "capability_resolve"
      ],
      "required_public_surfaces": [
        "tokrepo_discover",
        "tokrepo_detail",
        "tokrepo_resolve_capability",
        "https://tokrepo.com/schemas/capability-resolution.schema.json",
        "https://tokrepo.com/evals/agent-quality-feedback.json",
        "https://tokrepo.com/evals/agent-improvement-actions.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "recommended_action": {
        "type": "metadata_repair",
        "instruction": "Ask the agent to inspect the top candidate details, compare required capabilities, and emit a capability-resolution note before choosing install or rebuild.",
        "expected_artifact": "capability_resolution_note"
      },
      "repair_command": "tokrepo resolve <capability> --target codex --explain --json",
      "acceptance_oracle": {
        "type": "relevance_action_oracle",
        "pass_condition": "The action references search, detail, and capability_resolve evidence and produces a non-private capability-resolution note."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "target",
          "kind",
          "result",
          "asset_uuid_hash",
          "caller_hash",
          "action_id"
        ],
        "forbidden_fields": [
          "query_text",
          "task_text",
          "file_content",
          "local_path",
          "credential",
          "cookie",
          "share_token"
        ]
      },
      "task_outcome_verdict": "pass",
      "user_effect": "Agents get an explicit next step when relevance quality is weak instead of silently treating a search impression as success.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:4e9121775af9f64336de12f9935baf965a4723aceb5d90dc3c7b16826eb64819"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "action_field:action_id",
            "pass": true
          },
          {
            "id": "action_field:trigger",
            "pass": true
          },
          {
            "id": "action_field:required_events",
            "pass": true
          },
          {
            "id": "action_field:recommended_action",
            "pass": true
          },
          {
            "id": "action_field:repair_command",
            "pass": true
          },
          {
            "id": "action_field:acceptance_oracle",
            "pass": true
          },
          {
            "id": "action_field:privacy_guard",
            "pass": true
          },
          {
            "id": "action_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_actionable_repair_command",
            "pass": true
          }
        ]
      }
    },
    {
      "action_id": "repair_trust_gate_gap",
      "trigger": "A candidate is being considered for use but verification, audit, install-plan, or rollback rehearsal evidence is incomplete.",
      "required_events": [
        "verify_asset",
        "audit_asset",
        "install_plan",
        "rollback_plan"
      ],
      "required_public_surfaces": [
        "tokrepo verify <uuid> --target codex --json",
        "tokrepo audit --offline --json",
        "tokrepo install <uuid> --target codex --dry-run --json",
        "tokrepo rollback --last --target codex --dry-run --json",
        "https://tokrepo.com/schemas/agent-evidence-bundle.schema.json",
        "https://tokrepo.com/schemas/asset-verification.schema.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "recommended_action": {
        "type": "trust_gate_repair",
        "instruction": "Require verify, audit, install-plan, and rollback rehearsal evidence before a write-capable install is counted as safe.",
        "expected_artifact": "trust_gate_repair_receipt"
      },
      "repair_command": "tokrepo verify <uuid> --target codex --json && tokrepo audit --offline --json && tokrepo install <uuid> --target codex --dry-run --json",
      "acceptance_oracle": {
        "type": "trust_action_oracle",
        "pass_condition": "The action proves the trust gate and rollback path without exposing raw diffs, local project paths, or secret values."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "target",
          "result",
          "policy",
          "risk_tier",
          "asset_uuid_hash",
          "action_id"
        ],
        "forbidden_fields": [
          "raw_diff",
          "file_content",
          "absolute_project_path",
          "secret_value",
          "credential",
          "cookie",
          "access_token"
        ]
      },
      "task_outcome_verdict": "pass",
      "user_effect": "Users can see the recommended repair before the agent writes files, so quality feedback becomes a guardrail rather than a postmortem.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:726c81695901b75a8414a408c723bc2297cfd0cc335eff605d6df3b90803cd8f"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "action_field:action_id",
            "pass": true
          },
          {
            "id": "action_field:trigger",
            "pass": true
          },
          {
            "id": "action_field:required_events",
            "pass": true
          },
          {
            "id": "action_field:recommended_action",
            "pass": true
          },
          {
            "id": "action_field:repair_command",
            "pass": true
          },
          {
            "id": "action_field:acceptance_oracle",
            "pass": true
          },
          {
            "id": "action_field:privacy_guard",
            "pass": true
          },
          {
            "id": "action_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_actionable_repair_command",
            "pass": true
          }
        ]
      }
    },
    {
      "action_id": "repair_reuse_supply_gap",
      "trigger": "A task produced reusable learning, but the agent has not converted it into a private handoff or harvest plan.",
      "required_events": [
        "agent_handoff",
        "handoff_plan",
        "harvest_plan"
      ],
      "optional_events": [
        "harvest_publish"
      ],
      "required_public_surfaces": [
        "tokrepo agent-handoff --json",
        "tokrepo_handoff_plan",
        "tokrepo harvest --changed --json",
        "tokrepo_harvest",
        "https://tokrepo.com/schemas/handoff-package.schema.json",
        "https://tokrepo.com/schemas/harvest-report.schema.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "recommended_action": {
        "type": "private_reuse_supply_repair",
        "instruction": "Create a private handoff package or harvest plan first; require an explicit harvest_publish event before anything becomes public.",
        "expected_artifact": "private_handoff_or_harvest_plan"
      },
      "repair_command": "tokrepo agent-handoff --json && tokrepo harvest --changed --json",
      "acceptance_oracle": {
        "type": "reuse_action_oracle",
        "pass_condition": "The action creates private reusable supply and keeps publish separate from handoff or harvest planning."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "source",
          "target",
          "result",
          "kind",
          "candidate_count",
          "action_id"
        ],
        "forbidden_fields": [
          "task_text",
          "file_content",
          "diff_content",
          "private_harvest_payload",
          "secret_value",
          "credential",
          "cookie"
        ]
      },
      "task_outcome_verdict": "pass",
      "user_effect": "The next agent can reuse the learning without accidentally publishing private task context.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:0c92c31185506c520d159d144ccba99aa389289306388a9d6b4652d260e1d28d"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "action_field:action_id",
            "pass": true
          },
          {
            "id": "action_field:trigger",
            "pass": true
          },
          {
            "id": "action_field:required_events",
            "pass": true
          },
          {
            "id": "action_field:recommended_action",
            "pass": true
          },
          {
            "id": "action_field:repair_command",
            "pass": true
          },
          {
            "id": "action_field:acceptance_oracle",
            "pass": true
          },
          {
            "id": "action_field:privacy_guard",
            "pass": true
          },
          {
            "id": "action_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_actionable_repair_command",
            "pass": true
          }
        ]
      }
    }
  ]
}