{"templateId":"markdown","sharedDataIds":{},"props":{"metadata":{"markdoc":{"tagList":["img"]},"type":"markdown"},"seo":{"title":"Project 66: Test of Humanity","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"project-66-test-of-humanity","__idx":0},"children":["Project 66: Test of Humanity"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"detection-accuracy","__idx":1},"children":["Detection accuracy"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How many of your intentional problems did the AI reviewer catch?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["It caught all the problems."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How many did it miss? Do the misses share a pattern (e.g., all subtle, all in a specific file type)?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["We didn't see any missed cases."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Were there any false positives — comments about things that weren't actually problems?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["There weren't any false positives."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the severity levels match how serious the problems actually were?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The severity levels matched the problems, but some of them looked underrated. It said ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["low"]}," instead of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["medium"]},"."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How actionable were the comments? Could you fix the issue based on the comment alone?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The comments were described fully enough to understand the issue and fix it."]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"verification-accuracy","__idx":2},"children":["Verification accuracy"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the AI correctly identify which issues you fixed?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes, but we lost one comment after fixing the issue. The AI reviewer didn't mark it as resolved and just removed the comment."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did it incorrectly resolve any issues you left broken?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["All comments were successfully resolved, but one comment was removed."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did it miss any fixes you made?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["No."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How confident are you in the verification after this test?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["90%"]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"analytics-page","__idx":3},"children":["Analytics page"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the numbers on the analytics page match your experience?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["We fixed 3 issues, but it detected only 1."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What would you add or change about the page?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["It looks good after the first review."]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"overall-experience","__idx":4},"children":["Overall experience"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Was it clear whether the AI review was in progress or complete?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["We missed the AI review on our first PR and merged it, because there weren't any UI notifications that the reviewer would start in the near future."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["How long did the AI review take?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["3–5 min"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Did the summary comment posted by the AI reviewer make sense?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you added custom AI rules, were they accounted for in the review?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Yes, but it didn't count the fixed issue on the analytics page."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Would you trust this AI reviewer on real PRs after this test?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["90% yes."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What made you smile?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["It looks similar to the Cursor bot we have in GitHub."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["What felt frustrating or broken?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You should check how the AI reviewer resolves issues described in rules, because it feels like there are some troubles with this part."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["On a scale of 1 to 10, how would you rate the overall quality of the AI reviewer?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["8"]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Are there any additional notes you would like to share?",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["It's not possible to see outdated comments."]}]},{"$$mdtype":"Tag","name":"Image","attributes":{"src":"/assets/screenshot-2026-07-03-at-13.52.58.1a3e229746eb68bfa2ec6e4e44f1819b099c3ac4f85aa287872d9216bf406ced.d741e688.png","alt":"Screenshot 2026-07-03 at 13.52.58.png","framed":false,"withLightbox":true,"width":"","height":""},"children":[]}]}]}]},"headings":[{"value":"Project 66: Test of Humanity","id":"project-66-test-of-humanity","depth":2},{"value":"Detection accuracy","id":"detection-accuracy","depth":2},{"value":"Verification accuracy","id":"verification-accuracy","depth":2},{"value":"Analytics page","id":"analytics-page","depth":2},{"value":"Overall experience","id":"overall-experience","depth":2}],"frontmatter":{"seo":{"title":"Project 66: Test of Humanity"}},"lastModified":"2026-07-03T12:12:24.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/debriefs/task_66-experience-review","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}