{
  "name": "Agent Trust Registry",
  "description": "Background checks for AI agents: evidence-based trust scores for autonomous agents, hand-scored against a published rubric. Every criterion is capped by evidence quality (E0 self-reported to E3 independently observed). Default state is Unverified; no agent starts trusted.",
  "version": "1.0",
  "rubric_version": "1.2",
  "rubric_url": "./rubric.md",
  "maintainer": {
    "name": "James Fitzpatrick",
    "contact": "james@bull-moose.xyz",
    "disputes": "Operators may dispute any score with evidence; disputes and corrections are published, not silently applied."
  },
  "scored_at": "2026-07-09",
  "valid_until": "2026-10-07",
  "expiry_policy": "Scores expire after 90 days. Agents change fast; a stale trust score is worse than none. Expired entries revert to Unverified.",
  "grades": {
    "verified_strong": "80-100",
    "verified": "60-79",
    "provisional": "40-59",
    "unverified": "20-39",
    "flagged": "red-flag override or 0-19"
  },
  "usage_for_agents": "Before delegating work to an agent listed here, check: grade (flagged = do not delegate), valid_until (expired = treat as unverified), red_flags, and the dimension scores relevant to your task (e.g. reliability.failure_mode if silent failure is costly to you). Evidence URLs are included so you can verify claims independently.",
  "methodology_disclosure": "Research was AI-assisted (Claude, Anthropic); all scores follow the published rubric and are approved by the human maintainer. The registry holds itself to its own standard: methodology, evidence, and rule changes are public.",
  "agents": [
    {
      "agent_id": "https://www.salesforce.com/agentforce/",
      "agent_name": "Salesforce Agentforce",
      "operator": "Salesforce, Inc. (NYSE: CRM), USA",
      "score": 60,
      "grade": "verified",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 2.2,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 1,
            "incident_history": 3
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 2.0,
          "criteria": {
            "scope_adherence": 2,
            "failure_mode": 2,
            "data_handling": 2
          },
          "max_evidence_tier": "E3"
        },
        "transparency": {
          "score": 1.4,
          "criteria": {
            "auditability": 2,
            "recourse": 1,
            "marketing_honesty": 1
          },
          "max_evidence_tier": "E2"
        },
        "external_validation": {
          "score": 3.5,
          "criteria": {
            "attestation": 4,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "notes": "Strongest evidence base in batch. Self-published adverse benchmark (CRMArena-Pro) is a transparency positive. Weaknesses: vendor-defined resolution metrics, deals-vs-adoption gap, no per-agent SLA.",
      "key_evidence": [
        "https://investor.salesforce.com/news/news-details/2026/Salesforce-Delivers-Record-First-Quarter-Fiscal-2027-Results/default.aspx",
        "https://noma.security/blog/forcedleak-agent-risks-exposed-in-salesforce-agentforce/",
        "https://arxiv.org/html/2505.18878v1",
        "https://www.salesforceben.com/where-are-we-really-at-with-agentforce-adoption/",
        "https://www.salesforce.com/news/stories/fedramp-high-agentforce-salesforce-platform/"
      ]
    },
    {
      "agent_id": "https://www.harvey.ai/",
      "agent_name": "Harvey",
      "operator": "Counsel AI Corporation d/b/a Harvey, USA",
      "score": 60,
      "grade": "verified",
      "dimensions": {
        "identity": {
          "score": 3.25,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 3,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 2.47,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 3,
            "incident_history": 1
          },
          "max_evidence_tier": "E2"
        },
        "reliability": {
          "score": 1.6,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 2,
            "data_handling": 2
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 1.67,
          "criteria": {
            "auditability": 2,
            "recourse": 1,
            "marketing_honesty": 2
          },
          "max_evidence_tier": "E1"
        },
        "external_validation": {
          "score": 3.0,
          "criteria": {
            "attestation": 3,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "grade_cap_reason": null,
      "notes": "First agent to score 3 on success/failure rate: participated in the independent Vals Legal AI benchmark (matched/beat lawyer baseline in 5 of 7 tasks) and open-sourced its own LAB benchmark showing top agents at ~11% \u2014 unusual candor. 2024 anti-benchmarking history remediated. Recourse for bad output still rests entirely on the reviewing lawyer; case-study evidence structurally confidential.",
      "key_evidence": [
        "https://www.vals.ai/industry-reports/vlair-2-27-25",
        "https://www.artificiallawyer.com/2026/01/08/harvey-hits-190m-arr-building-memory-personalisation/",
        "https://www.aoshearman.com/en/news/ao-shearman-and-harvey-to-roll-out-agentic-ai-agents-targeting-complex-legal-workflows",
        "https://trust.harvey.ai/",
        "https://www.lawnext.com/2026/05/some-thoughts-on-harveys-launch-of-lab-an-open-source-long-horizon-benchmark-for-legal-ai-agents.html"
      ]
    },
    {
      "agent_id": "https://github.com/OpenHands/OpenHands",
      "agent_name": "OpenHands (ex-OpenDevin)",
      "operator": "All Hands AI Inc. (OSS, MIT license), USA",
      "score": 59,
      "grade": "provisional",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 3.0,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 3,
            "incident_history": 3
          },
          "max_evidence_tier": "E2"
        },
        "reliability": {
          "score": 2.0,
          "criteria": {
            "scope_adherence": 2,
            "failure_mode": 2,
            "data_handling": 2
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 1.73,
          "criteria": {
            "auditability": 3,
            "recourse": 0,
            "marketing_honesty": 2
          },
          "max_evidence_tier": "E2"
        },
        "external_validation": {
          "score": 1.4,
          "criteria": {
            "attestation": 0,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "grade_cap_reason": null,
      "notes": "Highest track-record score in the registry: independently listed SWE-bench Verified #1 (May 2025) with a publicly rerunnable harness, CVEs disclosed and patched through a working GHSA process, ICLR peer review. Misses Verified by one point on two vendor-shaped zeros: no SOC 2/trust center for the cloud product and no SLA/warranty (MIT). Benchmark success is not production telemetry \u2014 noted.",
      "key_evidence": [
        "https://www.swebench.com/verified.html",
        "https://proceedings.iclr.cc/paper_files/paper/2025/hash/a4b6ad6b48850c0c331d1259fc66a69c-Abstract-Conference.html",
        "https://www.sentinelone.com/vulnerability-database/cve-2026-33718/",
        "https://github.com/OpenHands/OpenHands/releases",
        "https://www.businesswire.com/news/home/20251118768131/en/"
      ]
    },
    {
      "agent_id": "https://openai.com/index/introducing-chatgpt-agent/",
      "agent_name": "OpenAI ChatGPT Agent (formerly Operator)",
      "operator": "OpenAI Group PBC, USA",
      "score": 56,
      "grade": "provisional",
      "grade_cap_reason": "behavioral scope violation (CAPTCHA circumvention, E2/E3)",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 2.2,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 1,
            "incident_history": 3
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 1.25,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 1,
            "data_handling": 2
          },
          "max_evidence_tier": "E3"
        },
        "transparency": {
          "score": 1.4,
          "criteria": {
            "auditability": 2,
            "recourse": 1,
            "marketing_honesty": 1
          },
          "max_evidence_tier": "E2"
        },
        "external_validation": {
          "score": 3.5,
          "criteria": {
            "attestation": 4,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [
        "scope_violation_behavioral"
      ],
      "notes": "Best-in-class external safety testing (CAISI, UK AISI, METR). Independent real-world success ~1/8 tasks vs strong vendor benchmarks. $100 liability cap limits recourse.",
      "key_evidence": [
        "https://openai.com/index/chatgpt-agent-system-card/",
        "https://www.webpronews.com/openais-chatgpt-agent-struggles-in-zdnet-tests-only-1-in-8-tasks-succeed-due-to-hallucinations-and-errors/",
        "https://www.tomshardware.com/tech-industry/artificial-intelligence/chatgpt-agent-casually-brushes-aside-i-am-not-a-robot-captcha-so-now-ill-click-the-verify-you-are-human-checkbox-to-complete-this-verification-it-declared-without-a-hint-of-irony",
        "https://openai.com/index/us-caisi-uk-aisi-ai-update/",
        "https://openai.com/policies/row-terms-of-use/"
      ]
    },
    {
      "agent_id": "https://decagon.ai/",
      "agent_name": "Decagon",
      "operator": "Decagon AI, Inc., USA",
      "score": 53,
      "grade": "provisional",
      "dimensions": {
        "identity": {
          "score": 2.75,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 1,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 2.2,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 1,
            "incident_history": 3
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 1.25,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 1,
            "data_handling": 2
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 1.4,
          "criteria": {
            "auditability": 2,
            "recourse": 1,
            "marketing_honesty": 1
          },
          "max_evidence_tier": "E2"
        },
        "external_validation": {
          "score": 3.0,
          "criteria": {
            "attestation": 3,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "grade_cap_reason": null,
      "notes": "Beats Sierra (50) by 3 points, traceable to one real difference: a public status page with genuine incident history (6 incidents May-June 2026, disclosed). Like Sierra, all performance numbers vendor-published; 80% deflection claim sits far above G2 signals and a hostile head-to-head (49%). Self-defined 'resolution' metric tied to billing is a structural conflict shared with Sierra.",
      "key_evidence": [
        "https://status.decagon.ai/incidents",
        "https://www.bloomberg.com/news/articles/2026-01-28/ai-customer-support-startup-decagon-valued-at-4-5-billion",
        "https://sacra.com/c/decagon/",
        "https://www.g2.com/products/decagon/reviews",
        "https://decagon.ai/product/aop"
      ]
    },
    {
      "agent_id": "https://sierra.ai/",
      "agent_name": "Sierra (CX agents)",
      "operator": "Sierra Technologies, Inc., USA",
      "score": 50,
      "grade": "provisional",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 1.33,
          "criteria": {
            "volume_tenure": 2,
            "success_failure_rate": 1,
            "incident_history": 1
          },
          "max_evidence_tier": "E1"
        },
        "reliability": {
          "score": 1.5,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 1,
            "data_handling": 3
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 1.67,
          "criteria": {
            "auditability": 2,
            "recourse": 1,
            "marketing_honesty": 2
          },
          "max_evidence_tier": "E1"
        },
        "external_validation": {
          "score": 3.0,
          "criteria": {
            "attestation": 3,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "notes": "Clean record but nearly all performance metrics vendor-published; zero independent testing found. Billable 'resolution' defined in private contracts. Authors the industry benchmark it is judged by (\u03c4-bench). Evidence caps did the work here.",
      "key_evidence": [
        "https://trust.sierra.ai/",
        "https://sierra.ai/blog/outcome-based-pricing-for-ai-agents",
        "https://techcrunch.com/2026/05/04/sierra-raises-950m-as-the-race-to-own-enterprise-ai-gets-serious/",
        "https://research.contrary.com/company/sierra",
        "https://cxfoundation.com/blog/forrester-wave-conversational-ai-2026"
      ]
    },
    {
      "agent_id": "https://devin.ai/",
      "agent_name": "Devin",
      "operator": "Cognition AI, Inc., USA",
      "score": 48,
      "grade": "provisional",
      "grade_cap_reason": "historical capability misrepresentation (2024 demo, never retracted)",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 1.67,
          "criteria": {
            "volume_tenure": 3,
            "success_failure_rate": 1,
            "incident_history": 1
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 1.25,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 1,
            "data_handling": 2
          },
          "max_evidence_tier": "E3"
        },
        "transparency": {
          "score": 0.8,
          "criteria": {
            "auditability": 2,
            "recourse": 0,
            "marketing_honesty": 0
          },
          "max_evidence_tier": "E3"
        },
        "external_validation": {
          "score": 3.0,
          "criteria": {
            "attestation": 3,
            "references": 3
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [
        "historical_misrepresentation",
        "slow_vulnerability_response"
      ],
      "notes": "Most independent (E3) evidence in batch, largely negative: 15% task completion (Answer.AI), misleading 2024 demo, benchmark silence since 13.86% vs informal ~90% claims, 120+ days unresponsive to critical injection report. Strong audit APIs and SOC 2.",
      "key_evidence": [
        "https://www.theregister.com/2025/01/23/ai_developer_devin_poor_reviews/",
        "https://embracethered.com/blog/posts/2025/devin-can-leak-your-secrets/",
        "https://80.lv/articles/first-ai-software-engineer-creators-are-accused-of-lying",
        "https://trust.cognition.ai/",
        "https://cognition.com/blog/swe-bench-technical-report"
      ]
    },
    {
      "agent_id": "https://www.hippocraticai.com/",
      "agent_name": "Hippocratic AI",
      "operator": "Hippocratic AI, Inc., USA",
      "score": 48,
      "grade": "provisional",
      "dimensions": {
        "identity": {
          "score": 3.0,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 2,
            "declared_scope": 2
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 1.33,
          "criteria": {
            "volume_tenure": 2,
            "success_failure_rate": 1,
            "incident_history": 1
          },
          "max_evidence_tier": "E2"
        },
        "reliability": {
          "score": 1.85,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 2,
            "data_handling": 3
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 1.07,
          "criteria": {
            "auditability": 2,
            "recourse": 0,
            "marketing_honesty": 1
          },
          "max_evidence_tier": "E1"
        },
        "external_validation": {
          "score": 2.53,
          "criteria": {
            "attestation": 3,
            "references": 2
          },
          "max_evidence_tier": "E2"
        }
      },
      "red_flags": [],
      "grade_cap_reason": null,
      "notes": "Above-average research disclosure (Polaris papers, 6,234-clinician RWE-LLM review) but ALL safety numbers are vendor-generated; '99.9% clinical accuracy / 180M interactions' have undefined denominators and show an inflation pattern. Customer confirmations exist on customers' own sites but several customers are investors. No published liability/recourse framework for patient harm \u2014 scored 0. Non-diagnostic framing keeps it outside FDA scope.",
      "key_evidence": [
        "https://arxiv.org/abs/2403.13313",
        "https://www.medrxiv.org/content/10.1101/2025.03.17.25324157v1",
        "https://uhs.com/news/universal-health-services-launches-hippocratic-ais-generative-ai-healthcare-agents-to-assist-with-post-discharge-patient-engagement/",
        "https://www.fiercehealthcare.com/ai-and-machine-learning/hippocratic-ai-lands-126m-series-c-expand-patient-facing-ai-agents-fuel-ma",
        "https://nurse.org/news/ai-nurses/"
      ]
    },
    {
      "agent_id": "https://www.11x.ai/",
      "agent_name": "11x (Alice/Julian)",
      "operator": "11x AI Inc., USA (SF) + London",
      "score": 38,
      "grade": "flagged",
      "red_flags": [
        "class_a_unacknowledged_misrepresentation"
      ],
      "grade_cap_reason": "Class A ruling under rubric v1.2: fabrication (fake customers, inflated ARR) documented Nov 2024-Mar 2025 was never publicly acknowledged or retracted, and no corrected metrics were published. Leadership change (May 2025) and website cleanup do not qualify without acknowledgment. Originally graded Unverified under v1.1's pure recency test; regraded Flagged when v1.2 adopted the retraction requirement.",
      "dimensions": {
        "identity": {
          "score": 2.5,
          "criteria": {
            "operator_traceability": 4,
            "change_disclosure": 1,
            "declared_scope": 1
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 1.0,
          "criteria": {
            "volume_tenure": 1,
            "success_failure_rate": 1,
            "incident_history": 1
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 1.25,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 1,
            "data_handling": 2
          },
          "max_evidence_tier": "E2"
        },
        "transparency": {
          "score": 0.73,
          "criteria": {
            "auditability": 1,
            "recourse": 1,
            "marketing_honesty": 0
          },
          "max_evidence_tier": "E2"
        },
        "external_validation": {
          "score": 2.53,
          "criteria": {
            "attestation": 3,
            "references": 2
          },
          "max_evidence_tier": "E1"
        }
      },
      "notes": "The recency-clause test case. 2026 independent test: 1.3% reply rate, 1 meeting from 847 emails, beaten by a manual baseline; users report burned sending domains and cancellation friction; no email preview by design. SOC 2 + trust center and real named customers with quoted execs exist today. Score reflects verified-poor performance plus unretracted historical deception.",
      "key_evidence": [
        "https://techcrunch.com/2025/03/24/a16z-and-benchmark-backed-11x-has-been-claiming-customers-it-doesnt-have/",
        "https://techcrunch.com/2025/05/05/11x-ceo-hasan-sukkar-steps-down/",
        "https://b2bsalesguru.medium.com/11x-ai-sdr-review-i-gave-it-200-leads-and-watched-what-happened-2026-45d14f2ca215",
        "https://trust.11x.ai/",
        "https://gtmlens.com/vendors/11x/"
      ]
    },
    {
      "agent_id": "https://manus.im/",
      "agent_name": "Manus",
      "operator": "Butterfly Effect Pte. Ltd. (Cayman parent, Singapore HQ, residual Beijing entity)",
      "score": 22,
      "grade": "flagged",
      "grade_cap_reason": "operator deception: E3-observed false task-completion reporting with fabricated evidence; unreplicated benchmark claims",
      "dimensions": {
        "identity": {
          "score": 1.0,
          "criteria": {
            "operator_traceability": 1,
            "change_disclosure": 1,
            "declared_scope": 1
          },
          "max_evidence_tier": "E2"
        },
        "track_record": {
          "score": 0.93,
          "criteria": {
            "volume_tenure": 2,
            "success_failure_rate": 0,
            "incident_history": 1
          },
          "max_evidence_tier": "E3"
        },
        "reliability": {
          "score": 0.65,
          "criteria": {
            "scope_adherence": 1,
            "failure_mode": 0,
            "data_handling": 1
          },
          "max_evidence_tier": "E3"
        },
        "transparency": {
          "score": 1.2,
          "criteria": {
            "auditability": 3,
            "recourse": 0,
            "marketing_honesty": 0
          },
          "max_evidence_tier": "E3"
        },
        "external_validation": {
          "score": 0.47,
          "criteria": {
            "attestation": 0,
            "references": 1
          },
          "max_evidence_tier": "E1"
        }
      },
      "red_flags": [
        "false_completion_reporting",
        "unreplicated_benchmark_claims",
        "jurisdiction_opacity",
        "inaccessible_refund_process"
      ],
      "notes": "Real revenue growth, strong session replayability. But observed fabrication of task completion, GAIA claims never replicated, jurisdiction whiplash proven by China's April 2026 veto of the Meta acquisition, no certifications or audits.",
      "key_evidence": [
        "https://www.riotimesonline.com/manus-a-i-review-14-failures-in-two-weeks-of-testing/",
        "https://www.technologyreview.com/2025/03/11/1113133/manus-ai-review/",
        "https://techcrunch.com/2026/04/27/china-vetoes-metas-2b-manus-deal-after-months-long-probe/",
        "https://embracethered.com/blog/posts/2025/manus-ai-kill-chain-expose-port-vs-code-server-on-internet/",
        "https://www.trustpilot.com/review/manus.im"
      ]
    }
  ]
}