{
  "schema_version": "2026-05-28.agent-quality-feedback.v1",
  "name": "TokRepo Agent Quality Feedback Eval",
  "canonical_url": "https://tokrepo.com/evals/agent-quality-feedback.json",
  "updated_at": "2026-05-28",
  "generated_by": "node scripts/run_agent_quality_feedback.mjs --json",
  "purpose": "Public deterministic contract for proving that TokRepo closes the loop from agent relevance, trust-gated use, and private improvement supply without publishing task text, local paths, file contents, raw diffs, secrets, or private harvest payloads.",
  "quality_feedback_contract": {
    "public_feedback_api": "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel",
    "event_ingestion_api": "https://api.tokrepo.com/api/v1/tokenboard/agent/events",
    "required_live_window_days": 14,
    "required_live_event_keys": [
      "mcp_search",
      "mcp_detail",
      "capability_resolve",
      "verify_asset",
      "audit_asset",
      "agent_handoff",
      "harvest_plan"
    ],
    "optional_live_event_keys": [
      "handoff_plan",
      "harvest_publish",
      "install_plan"
    ],
    "relevance_contract": [
      "tokrepo_discover",
      "tokrepo_detail",
      "tokrepo_resolve_capability",
      "capability-resolution.schema.json"
    ],
    "trust_gate_contract": [
      "tokrepo verify <uuid> --target codex --json",
      "tokrepo audit --offline --json",
      "tokrepo install <uuid> --target codex --dry-run --json",
      "agent-evidence-bundle.schema.json"
    ],
    "improvement_contract": [
      "tokrepo agent-handoff --json",
      "tokrepo_handoff_plan",
      "tokrepo harvest --changed --json",
      "tokrepo_harvest"
    ],
    "completion_boundary": "Quality feedback is not a page view, search impression, or install count. It requires relevance resolution, trust-gate evidence, and post-task handoff or harvest supply; harvest_publish remains explicit and separately counted.",
    "privacy_boundary": "Only anonymous aggregate counters, hashed identifiers, public schemas, and deterministic coverage metadata may be published. The contract must not expose query text, task text, file contents, raw diffs, local paths, credentials, cookies, share tokens, or private harvest payloads.",
    "live_check_command": "WEB_BASE=https://tokrepo.com API_BASE=https://api.tokrepo.com node scripts/run_agent_quality_feedback.mjs --check-live"
  },
  "metrics": {
    "stages_total": 3,
    "stages_passed": 3,
    "quality_feedback_stage_pass_pct": 100,
    "required_live_event_coverage_pct": 100,
    "trust_gate_coverage_pct": 100,
    "improvement_supply_coverage_pct": 100,
    "privacy_guard_coverage_pct": 100
  },
  "quality_stages": [
    {
      "stage_id": "relevance_resolution_feedback",
      "task": "Prove that agents can move from broad search to detail inspection and capability resolution before choosing an asset.",
      "required_events": [
        "mcp_search",
        "mcp_detail",
        "capability_resolve"
      ],
      "required_public_surfaces": [
        "tokrepo_discover",
        "tokrepo_resolve_capability",
        "tokrepo_detail",
        "https://tokrepo.com/schemas/capability-resolution.schema.json",
        "https://tokrepo.com/evals/agent-quality-feedback.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "feedback_signal": {
        "source": "anonymous_search_detail_resolve_funnel",
        "required_live_keys": [
          "mcp_search",
          "mcp_detail",
          "capability_resolve"
        ],
        "interpretation": "Quality feedback starts when agents inspect candidates deeply enough to resolve a capability instead of treating search impressions as successful reuse."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "target",
          "kind",
          "result",
          "asset_uuid_hash",
          "caller_hash"
        ],
        "forbidden_fields": [
          "query_text",
          "task_text",
          "local_path",
          "file_content",
          "credential",
          "cookie",
          "share_token"
        ]
      },
      "outcome_oracle": {
        "type": "relevance_before_install_guard",
        "pass_condition": "The public contract proves search, detail, and resolve signals without exposing the private task prompt or local project context."
      },
      "task_outcome_verdict": "pass",
      "user_effect": "Agents get a measurable relevance loop before installing, so TokRepo can improve asset matching instead of optimizing only for impressions.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:f167f839e9e987e4d216501e66b4bd0a12b5aa8140b6717b8c44ee52a3140b5a"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "stage_field:stage_id",
            "pass": true
          },
          {
            "id": "stage_field:task",
            "pass": true
          },
          {
            "id": "stage_field:required_events",
            "pass": true
          },
          {
            "id": "stage_field:required_public_surfaces",
            "pass": true
          },
          {
            "id": "stage_field:feedback_signal",
            "pass": true
          },
          {
            "id": "stage_field:privacy_guard",
            "pass": true
          },
          {
            "id": "stage_field:outcome_oracle",
            "pass": true
          },
          {
            "id": "stage_field:task_outcome_verdict",
            "pass": true
          },
          {
            "id": "stage_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_outcome_oracle_pass_condition",
            "pass": true
          }
        ]
      }
    },
    {
      "stage_id": "trust_gate_feedback",
      "task": "Prove that selected assets pass verification, audit, and install-plan gates before write-capable use.",
      "required_events": [
        "verify_asset",
        "audit_asset",
        "install_plan"
      ],
      "required_public_surfaces": [
        "tokrepo verify <uuid> --target codex --json",
        "tokrepo audit --offline --json",
        "tokrepo install <uuid> --target codex --dry-run --json",
        "https://tokrepo.com/schemas/agent-evidence-bundle.schema.json",
        "https://tokrepo.com/schemas/asset-verification.schema.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "feedback_signal": {
        "source": "verify_audit_install_plan_funnel",
        "required_live_keys": [
          "verify_asset",
          "audit_asset",
          "install_plan"
        ],
        "interpretation": "Quality is counted only when trust evidence and install planning are inspectable before the agent mutates a project."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "target",
          "result",
          "policy",
          "risk_tier",
          "asset_uuid_hash"
        ],
        "forbidden_fields": [
          "raw_diff",
          "file_content",
          "absolute_project_path",
          "secret_value",
          "credential",
          "cookie",
          "access_token"
        ]
      },
      "outcome_oracle": {
        "type": "trust_before_write_guard",
        "pass_condition": "verify_asset, audit_asset, and install_plan prove the quality gate while the public eval forbids raw diffs, secrets, and local paths."
      },
      "task_outcome_verdict": "pass",
      "user_effect": "A user can see that TokRepo measures quality at the trust gate, not only after an install is already complete.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:c4addd9f40c102e940f289ec18f1f0eef4b8abf08770a0a9231647ecc74aed76"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "stage_field:stage_id",
            "pass": true
          },
          {
            "id": "stage_field:task",
            "pass": true
          },
          {
            "id": "stage_field:required_events",
            "pass": true
          },
          {
            "id": "stage_field:required_public_surfaces",
            "pass": true
          },
          {
            "id": "stage_field:feedback_signal",
            "pass": true
          },
          {
            "id": "stage_field:privacy_guard",
            "pass": true
          },
          {
            "id": "stage_field:outcome_oracle",
            "pass": true
          },
          {
            "id": "stage_field:task_outcome_verdict",
            "pass": true
          },
          {
            "id": "stage_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_outcome_oracle_pass_condition",
            "pass": true
          }
        ]
      }
    },
    {
      "stage_id": "improvement_supply_feedback",
      "task": "Prove that post-task feedback can become private handoff or harvest supply and only publishes through an explicit event.",
      "required_events": [
        "agent_handoff",
        "handoff_plan",
        "harvest_plan",
        "harvest_publish"
      ],
      "required_public_surfaces": [
        "tokrepo agent-handoff --json",
        "tokrepo_handoff_plan",
        "tokrepo harvest --changed --json",
        "https://tokrepo.com/schemas/handoff-package.schema.json",
        "https://tokrepo.com/schemas/harvest-report.schema.json",
        "https://api.tokrepo.com/api/v1/tokenboard/agent/funnel"
      ],
      "feedback_signal": {
        "source": "handoff_harvest_publish_funnel",
        "required_live_keys": [
          "agent_handoff",
          "handoff_plan",
          "harvest_plan"
        ],
        "optional_live_keys": [
          "harvest_publish"
        ],
        "interpretation": "Improvement feedback is reusable supply when handoff or harvest plans exist; publish remains explicit and separately measured."
      },
      "privacy_guard": {
        "publishable_fields_only": [
          "event",
          "source",
          "target",
          "result",
          "kind",
          "candidate_count"
        ],
        "forbidden_fields": [
          "task_text",
          "file_content",
          "diff_content",
          "private_harvest_payload",
          "secret_value",
          "credential",
          "cookie"
        ]
      },
      "outcome_oracle": {
        "type": "private_improvement_before_publish_guard",
        "pass_condition": "handoff_plan and harvest_plan prove self-improvement supply while harvest_publish remains an explicit, separately counted event."
      },
      "task_outcome_verdict": "pass",
      "user_effect": "TokRepo can close the loop from asset use to better future assets without automatically publishing private work.",
      "evidence_digest": {
        "algorithm": "sha256",
        "digest": "sha256:efcac0e7abb8b212e655a9b458a93d3df390058ae0e3d545659732d9232a4f2e"
      },
      "coverage": {
        "pass": true,
        "checks": [
          {
            "id": "stage_field:stage_id",
            "pass": true
          },
          {
            "id": "stage_field:task",
            "pass": true
          },
          {
            "id": "stage_field:required_events",
            "pass": true
          },
          {
            "id": "stage_field:required_public_surfaces",
            "pass": true
          },
          {
            "id": "stage_field:feedback_signal",
            "pass": true
          },
          {
            "id": "stage_field:privacy_guard",
            "pass": true
          },
          {
            "id": "stage_field:outcome_oracle",
            "pass": true
          },
          {
            "id": "stage_field:task_outcome_verdict",
            "pass": true
          },
          {
            "id": "stage_field:user_effect",
            "pass": true
          },
          {
            "id": "has_live_funnel_api",
            "pass": true
          },
          {
            "id": "has_privacy_forbidden_fields",
            "pass": true
          },
          {
            "id": "has_outcome_oracle_pass_condition",
            "pass": true
          }
        ]
      }
    }
  ]
}