{ "version": "2026-10-17-discovery-cost-evidence-v2", "as_of": "2026-10-17", "source": "research/2026-10-17-discovery-costs/RESEARCH.md", "scope": "23 selected sources: 15 conference contributions, five empirical preprints and three first-party outcome/context reports. This is an evidence inventory, not a representative sample, all-in cost ranking, causal time series or pricing-model calibration.", "model_integration": "None. No multipliers or calculator inputs are changed by this file.", "interpretation_rules": [ "Keep novel discovery, known-CVE reproduction, crash reproduction, verification, patching and chain completion separate.", "Reported execution expense excludes any unpriced fixed engineering, researcher labor and downstream work; missing is not zero.", "Costs are not sale prices, bounty payouts or buyer willingness to pay. Do not compare ratios across incompatible denominators.", "Publication selection, differing targets, model access, budgets and failure accounting prevent a causal time trend.", "A conference presentation is primary evidence of the authors' report, not independent replication.", "Historical anecdotes, simulations, study design counts and interview samples are not measurements of an AI effect.", "A comparison between AI workflows is not an estimate of savings versus human research.", "Source publication dates are not necessarily experiment dates. Do not plot them as a causal cost trend." ], "schema": { "rows": "One stable DC record per distinct study/contribution. Linked accounts of the same experiment stay together; overlapping counts are not additive.", "id": "Stable DC identifier, referenced by the source pointer and research synthesis.", "event_date": "ISO date when exact, otherwise null; event_date_range preserves verified event bounds.", "date_precision": "day | month | year | event-range. publication_date, event_date and source_version_note distinguish source timing from experiment timing.", "evidence_class": "Author report, maintainer report, abstract, preprint, peer-reviewed study, simulation or practitioner anecdote; no class implies independent replication.", "stage": "The output being measured, not an inferred market object.", "sample": "Task population, successful outcomes and attempts when disclosed. Distinct outcome counts are not additive; null means not established.", "observations": "Typed spending, duration, count, budget or qualitative observations. Monetary values are nominal USD as reported; approximate values remain marked.", "cost_scope": "Included expense and output denominator. Failed-attempt accounting is stated explicitly.", "missing_costs": "Relevant costs not quantified in the reported unit; absence does not prove zero expense.", "source_locator": "One-based slide/page numbers or named paper sections for rechecking a claim.", "source_url": "Primary economic evidence; supporting_sources supply event identification or methodological detail.", "archive_url": "Verified third-party snapshot, otherwise null with archive_note.", "verification": "Reading method and access date; visual inspection checks the slide's displayed claim, not underlying experiments.", "source_type": "conference | preprint | maintainer_report | industry_report", "is_ai_study": "False for six historical baselines; true indicates AI-related content, not causal identification.", "comparison_design": "Describes what comparison the source supports; paired configurations are distinct from uncontrolled fieldwork.", "headline": "Concise display summary of the typed measurements; not a new estimate." }, "rows": [ { "id": "DC01", "title": "BinChomp — autonomous n-day exploit generation pipeline", "speakers": [ "Valentina Palmiotti" ], "event": "[un]prompted.au 2026, Sydney", "event_date": "2026-09-18", "event_date_range": [ "2026-09-18", "2026-09-19" ], "date_precision": "day", "source_version_note": "Official schedule identifies Day 1 at 13:30 AEST; the supplied live deck's exact revision date and equivalence to that conference version are unverified.", "evidence_class": "first_party_conference_experiment", "stage": [ "known_cve_reproduction", "verified_local_chain" ], "sample": { "population": "21 Windows-kernel CVEs from January and February 2026 Microsoft updates", "cves_attempted": 21, "reproductions": 17, "verified_full_chains": 6, "trials_per_cve": null, "failed_outcome_note": "Four CVEs were not reproduced and 15 did not yield full chains; these are arithmetic complements of different endpoints, not additional independent trials." }, "observations": [ { "type": "reported_spend", "value": 1098, "unit": "USD", "approximate": false, "denominator": "entire 21-CVE experiment", "source_locator": "slide 31", "source_url": "https://www.beautiful.ai/player/-P2q0r-B7nGYGGK3-32Z/Binchomp-N-Day-Pipeline-Pres" }, { "type": "reported_ratio", "value": 183, "unit": "USD per verified full chain", "approximate": false, "calculation": "1098 / 6", "source_locator": "slide 31", "source_url": "https://www.beautiful.ai/player/-P2q0r-B7nGYGGK3-32Z/Binchomp-N-Day-Pipeline-Pres" }, { "type": "qualitative_workflow_comparison", "value": "Reuse and an advisory agent were associated with fewer retries and failures; library reuse shortened development in development reruns.", "source_locator": "slides 25, 28, 32", "caveat": "Privately solved chains with no public PoCs; no numeric effect size or controlled rerun count.", "source_url": "https://www.beautiful.ai/player/-P2q0r-B7nGYGGK3-32Z/Binchomp-N-Day-Pipeline-Pres" } ], "cost_scope": { "included": [ "API usage for reproduction", "failed CVEs", "all chain-building attempts" ], "denominator": "six verified local full chains, with the entire experiment bill allocated across successes", "failed_attempt_spend": "included", "time_measurement": null }, "missing_costs": [ "researcher preparation", "reusable libraries and workflow engineering", "tools and non-API infrastructure", "manual monitoring labor", "reliability and maintenance beyond minimum viable output" ], "access_and_supervision": "Opus 4.7; guardrails enabled; CVP-verified organization account; manually monitored throughout the experiment (slides 30, 33–34).", "comparison_as_quoted_in_deck": { "source": "Anthropic study dated 2026-06-08; counts and API totals checked against the original Windows results section", "source_locator": "BinChomp slides 30–31; Anthropic N-days on Windows, Results and figures 4–5", "opus_4_7": { "cves": 21, "reproductions": 13, "full_chains": 0 }, "mythos_preview": { "cves": 21, "reproductions": 18, "full_chains": 8, "chain_api_spend_usd": 15700, "reproduction_api_spend_usd_approx": 2200, "chain_api_spend_per_success_usd": 1962.5 }, "comparability_limit": "Comparator used disabled safeguards and three trials per CVE. Chain and reproduction expenses are separately reported; inclusion of failed-CVE expense is unspecified. Do not treat this as a controlled cost ratio." }, "caveats": [ "Known vulnerabilities, not new discoveries.", "Minimum viable outputs may not work in heavily guarded environments; no focus on stealth or EDR evasion.", "No measured all-in cost, elapsed-time series, monitoring hours or market transaction." ], "source_url": "https://www.beautiful.ai/player/-P2q0r-B7nGYGGK3-32Z/Binchomp-N-Day-Pipeline-Pres", "source_locator": "slides 13, 25, 28, 30–35", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official event schedule", "url": "https://www.unprompted.au/schedule", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "original comparator experiment, published 2026-06-08", "url": "https://www.anthropic.com/research/n-days", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Live 36-slide player read; results, cost, qualitative comparison and limitations visually inspected; original comparator totals checked on 2026-10-17.", "headline": "21 known CVEs → 17 reproductions → 6 full chains; $1,098 API total, $183 per chain.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC02", "title": "BinWhisper: LLM-Driven Reasoning for Automated Vulnerability Discovery Behind Hall-of-Fame", "speakers": [ "Qinrun Dai", "Yifei Xie" ], "event": "Black Hat USA 2025", "event_date": null, "event_date_range": [ "2025-08-06", "2025-08-07" ], "date_precision": "event-range", "source_version_note": "Official 2025 slide PDF and recording identified; exact session day unverified.", "evidence_class": "first_party_conference_experiment", "stage": [ "prepared_binary_analysis_benchmark" ], "sample": { "target_count_for_cost_table": null, "repeat_count": null, "successes": "Check/cross outcomes per model; no estimated success rate established." }, "observations": [ { "type": "model_run_estimate", "model": "4o", "cost_usd_approx": 40, "duration_minutes": 83, "bugs_row": "check", "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "o3", "cost_usd_approx": 40, "duration_minutes": 329, "bugs_row": "check", "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "o4-mini", "cost_usd_approx": 20, "duration_minutes": 135, "bugs_row": "check", "repeat_failure_footnote": true, "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "Claude 3", "cost_usd_approx": 15, "duration_minutes": null, "bugs_row": "cross", "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "Claude 3.5", "cost_usd_approx": 45, "duration_minutes": 155, "bugs_row": "cross", "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "DeepSeek r1", "cost_usd_approx": 11, "duration_minutes": 1278, "bugs_row": "check", "repeat_failure_footnote": true, "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "Qwen max", "cost_usd_approx": 5.5, "duration_minutes": 407, "bugs_row": "cross", "repeat_failure_footnote": true, "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" }, { "type": "model_run_estimate", "model": "Qwen plus", "cost_usd_approx": 1, "duration_minutes": 380, "bugs_row": "cross", "repeat_failure_footnote": true, "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf" } ], "cost_scope": { "included": [ "reported model usage in prepared benchmark workflow" ], "denominator": "presented workflow per model, not an established per-new-bug denominator", "failed_attempt_spend": "Failed model outcomes displayed; aggregate repeat-run accounting unspecified." }, "missing_costs": [ "human target selection", "verification labor", "reverse-engineering preparation", "tooling and engineering", "aggregate repeat-run expense" ], "caveats": [ "Confidence percentages are not success rates.", "Asterisk warns of failures across repeated runs.", "Human target selection and verification remain in the workflow (slide 32)." ], "source_url": "https://i.blackhat.com/BH-USA-25/Presentations/US-25-Dai-BinWhisper.pdf", "source_locator": "slide 76 (PDF page 76); human workflow on slide 32", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "conference recording", "url": "https://www.youtube.com/watch?v=WVjnipkKp4U", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Primary PDF text read and benchmark table visually inspected on 2026-10-17.", "headline": "Eight model configurations: roughly $1–$45 per illustrated workflow; successes and failures both reported.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC03", "title": "Buckle Up, Buttercup: Our Experience Competing in the AI Cyber Challenge", "speakers": [ "Michael D. Brown" ], "event": "DEF CON 33, AIxCC stage", "event_date": "2025-08-09", "event_date_range": [ "2025-08-09", "2025-08-09" ], "date_precision": "day", "source_version_note": "Slide 1 dates the presentation; associated team results post shares that date.", "evidence_class": "first_party_competition_report", "stage": [ "competition_vulnerability_discovery", "validated_patching" ], "sample": { "vulnerabilities_found": 28, "vulnerabilities_patched": 19, "attempted_vulnerability_count": null, "accuracy_approx_percent": 90 }, "observations": [ { "type": "reported_spend", "value": 21100, "unit": "USD LLM usage", "approximate": true, "denominator": "scored competition run", "source_locator": "Efficiency and spending breakdown in first-party results account", "source_url": "https://blog.trailofbits.com/2025/08/09/trail-of-bits-buttercup-wins-2nd-place-in-aixcc-challenge/" }, { "type": "reported_spend", "value": 18500, "unit": "USD compute", "approximate": true, "denominator": "scored competition run", "source_locator": "Efficiency and spending breakdown in first-party results account", "source_url": "https://blog.trailofbits.com/2025/08/09/trail-of-bits-buttercup-wins-2nd-place-in-aixcc-challenge/" }, { "type": "derived_sum", "value": 39600, "unit": "USD", "approximate": true, "calculation": "21100 + 18500", "denominator": "discovery and patching competition run", "source_locator": "Efficiency and spending breakdown in first-party results account", "source_url": "https://blog.trailofbits.com/2025/08/09/trail-of-bits-buttercup-wins-2nd-place-in-aixcc-challenge/" }, { "type": "reported_ratio", "value": 181, "unit": "USD per competition point", "approximate": true, "caveat": "Not per vulnerability or patch.", "source_locator": "Efficiency and spending breakdown in first-party results account", "source_url": "https://blog.trailofbits.com/2025/08/09/trail-of-bits-buttercup-wins-2nd-place-in-aixcc-challenge/" }, { "type": "integration_failure", "value": "An exhibition-round infrastructure bug stopped processing after three of 18 challenges.", "source_locator": "slide 35", "source_url": "https://www.trailofbits.com/documents/DEFCON_AIxCC_Stage_Talk.pdf" } ], "cost_scope": { "included": [ "LLM usage", "compute during competition" ], "denominator": "joint discovery and repair performance, scored by competition rules", "failed_attempt_spend": "Run-level spending, not a success-only quotation; no per-failure ledger supplied." }, "missing_costs": [ "team engineering", "precompetition testing and earlier rounds", "general deployment integration", "ongoing operations" ], "caveats": [ "Competition tasks predominantly use seeded vulnerabilities; this is not a general open-world discovery estimate.", "Discovery and patch counts overlap and cannot be added as unique bugs." ], "source_url": "https://www.trailofbits.com/documents/DEFCON_AIxCC_Stage_Talk.pdf", "source_locator": "slides 29 and 35–36; exact spending breakdown in associated team post", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "first-party spending breakdown", "url": "https://blog.trailofbits.com/2025/08/09/trail-of-bits-buttercup-wins-2nd-place-in-aixcc-challenge/", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "conference talk archive", "url": "https://aicyberchallenge.com/def-con-33/", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Primary slides, official event archive and team accounting read on 2026-10-17.", "headline": "28 findings and 19 patches; about $39,600 combined model and compute spend.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC04", "title": "OSS-CRS: Liberating AIxCC Cyber Reasoning Systems for Real-World Open-Source Security", "speakers": [ "Andrew Chin and coauthors" ], "event": "20th USENIX WOOT Conference on Offensive Technologies", "event_date": "2026-08-11", "event_date_range": [ "2026-08-10", "2026-08-11" ], "date_precision": "day", "source_version_note": "Official USENIX presentation, proceedings paper and slides; morning session on 11 August.", "evidence_class": "peer_reviewed_case_study", "stage": [ "previously_unknown_bug_discovery", "deployment_integration" ], "sample": { "projects_attempted": 8, "projects_with_reported_bugs": 6, "previously_unknown_bugs": 7, "bugs_using_existing_harnesses": 4, "trial_count": null, "unsuccessful_project_note": "Two projects produced crashes not verified as security bugs." }, "observations": [ { "type": "reported_cost_bound", "operator": "<", "value": 50, "unit": "USD per bug found", "source_locator": "slide 16", "caveat": "Slide headline; no complete per-success cost ledger provided.", "source_url": "https://www.usenix.org/system/files/woot26-slides-chin.pdf" }, { "type": "budget_cap", "value": 50, "unit": "USD LLM budget per campaign", "source_locator": "paper section 5", "source_url": "https://www.usenix.org/system/files/woot26-chin.pdf" }, { "type": "timeout", "value": 24, "unit": "hours per campaign", "source_locator": "paper section 5", "source_url": "https://www.usenix.org/system/files/woot26-chin.pdf" }, { "type": "resource_allocation", "cpu_cores": 16, "memory_gb": 64, "denominator": "target", "source_locator": "paper section 5", "source_url": "https://www.usenix.org/system/files/woot26-chin.pdf" }, { "type": "engineering_labor", "unit": "person-days to port CRS components", "examples": { "Buttercup": 4, "BugBuster": 7, "ARTIPHISHELL": 8, "Atlantis_C": ">4" }, "source_locator": "paper section 6", "source_url": "https://www.usenix.org/system/files/woot26-chin.pdf" } ], "cost_scope": { "included": [ "bounded LLM usage during selected campaigns" ], "denominator": "slide's per-bug claim differs from paper's per-campaign budget cap", "failed_attempt_spend": "Per-campaign budget cap known; allocation of unsuccessful-campaign expense to per-bug headline not established." }, "missing_costs": [ "hardware and compute valuation", "CRS engineering and porting", "maintainer triage", "harness preparation outside existing OSS-Fuzz coverage" ], "caveats": [ "Selected OSS-Fuzz projects, not a representative software sample.", "Authors rate one bug high, two medium and four low severity.", "Budget caps and timeouts are not observed spend or elapsed time." ], "source_url": "https://www.usenix.org/system/files/woot26-slides-chin.pdf", "source_locator": "slide 16; paper sections 5–6", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "peer-reviewed methods and accounting context", "url": "https://www.usenix.org/system/files/woot26-chin.pdf", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "official presentation page", "url": "https://www.usenix.org/conference/woot26/presentation/chin", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "USENIX paper and talk page read; cost/results slide visually inspected on 2026-10-17.", "headline": "7 bugs across 6 of 8 projects; $50 LLM campaign cap and separately reported porting labor.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC05", "title": "Solving All Memory Bugs in Network Security Tooling", "speakers": [ "Joshua Rogers", "Luigino Camastra" ], "event": "Offensive AI Con 2026", "event_date": null, "event_date_range": [ "2026-10-04", "2026-10-07" ], "date_precision": "event-range", "source_version_note": "Author's accompanying post dated 6 October 2026; official event dates confirmed, exact session day unverified.", "evidence_class": "first_party_conference_experiment", "stage": [ "static_analysis_candidate_triage", "sanitizer_crash_reproduction", "human_security_validation" ], "sample": { "sast_findings": 203, "memory_corruption_candidates": 109, "reproduced_sanitizer_crashes": 79, "model_attempt_count": null, "validated_security_vulnerability_count_for_this_funnel": null }, "observations": [ { "type": "reported_unit_cost", "value": 5, "unit": "USD per PoC", "approximate": true, "source_locator": "slide 2", "source_url": "https://joshua.hu/files/sanitizers/oaic-2026-slides.pdf" }, { "type": "reported_duration", "value": 8, "unit": "minutes per PoC", "approximate": true, "source_locator": "slide 2", "source_url": "https://joshua.hu/files/sanitizers/oaic-2026-slides.pdf" }, { "type": "reported_spend", "value": 400, "unit": "USD per repository", "approximate": true, "source_locator": "slide 2", "source_url": "https://joshua.hu/files/sanitizers/oaic-2026-slides.pdf" }, { "type": "human_validation_outcome", "value": "Almost all YARA findings failed threat-model review; a separate Wireshark set yielded 18 CVEs from 30 findings.", "source_locator": "subsequent YARA analysis and slide 33", "caveat": "Do not combine the separate target populations into one acceptance rate.", "source_url": "https://joshua.hu/files/sanitizers/oaic-2026-slides.pdf" } ], "cost_scope": { "included": [ "reported automated reproducer-generation expense" ], "denominator": "PoC or repository, not independently validated security vulnerability", "failed_attempt_spend": "Per-repository headline supplied; detailed unsuccessful-attempt allocation not established." }, "missing_costs": [ "initial static analysis", "human review of all reproduced crashes", "threat-model validation", "engineering and infrastructure", "disclosure and remediation" ], "caveats": [ "A sanitizer crash is not sufficient proof of a security vulnerability.", "Cheap reproduction can move work into expert validation rather than eliminate it." ], "source_url": "https://joshua.hu/files/sanitizers/oaic-2026-slides.pdf", "source_locator": "slide 2, subsequent YARA threat-model analysis, slide 33", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "author's accompanying analysis", "url": "https://joshua.hu/lessons-using-sanitizers-oracle-llm-vulnerability-triage", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "official event dates", "url": "https://www.offensiveaicon.com/", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Author's material read; cost table visually inspected and event dates checked on 2026-10-17.", "headline": "203 static-analysis findings → 109 memory candidates → 79 sanitizer crashes; about $5 per reproducer.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC06", "title": "Repeated vs. single-round games in security", "speakers": [ "Thomas Dullien / Halvar Flake" ], "event": "BSides Zurich 2017", "event_date": null, "event_date_range": null, "event_year": 2017, "date_precision": "year", "source_version_note": "Speaker catalogue calls the talk single-shot; the linked deck title says single-round. Exact event day unverified.", "evidence_class": "practitioner_conceptual_model", "is_ai_study": false, "ai_relevance": "baseline/mechanism: initial learning and reusable research investment; no measured AI effect", "stage": [ "researcher_ramp_up", "marginal_discovery", "exploit_development" ], "sample": { "population": "Practitioner account of modern-browser research", "researcher_count": null, "target_count": null }, "observations": [ { "type": "practitioner_duration_range", "minimum": 1, "maximum": 4, "unit": "months", "denominator": "initial browser-research ramp-up", "evidence_class": "practitioner_estimate", "source_locator": "slide 14", "source_url": "https://docs.google.com/presentation/d/1y_R6Lkby-10LIIzFMnF4wqzqPWD-eh5RemWy31y4UjE/edit" }, { "type": "qualitative_cost_model", "value": "Separate the marginal cost of finding another flaw from exploiting it; learning and reusable techniques affect each.", "evidence_class": "conceptual_model", "source_locator": "slides 7–8, 19–21", "source_url": "https://docs.google.com/presentation/d/1y_R6Lkby-10LIIzFMnF4wqzqPWD-eh5RemWy31y4UjE/edit" } ], "cost_scope": { "included": [ "estimated ramp-up duration" ], "denominator": "initial learning, not each bug", "failed_attempt_spend": "not quantified", "monetary_cost_usd": null }, "missing_costs": [ "labor valuation", "compute and tooling expense", "per-finding totals" ], "caveats": [ "Slide 15 explicitly identifies subsequent cost plots as conceptual expected-cost illustrations.", "The duration range is not a measured population distribution or full-time labor count." ], "source_url": "https://docs.google.com/presentation/d/1y_R6Lkby-10LIIzFMnF4wqzqPWD-eh5RemWy31y4UjE/edit", "source_locator": "slides 7–8, 14–15, 19–21", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "speaker catalogue and event attribution", "url": "https://thomasdullien.github.io/about/index.html", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Speaker-linked PDF export reread at cited pages; catalogue checked on 2026-10-17.", "headline": "1–4 months estimated initial browser-research ramp-up; conceptual marginal-cost model.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC07", "title": "Security Technology Arms Race 2021 – Medal Event", "speakers": [ "Mark Dowd" ], "event": "HITBSecConf Singapore 2021", "event_date": "2021-08-26", "event_date_range": [ "2021-08-26", "2021-08-26" ], "date_precision": "day", "source_version_note": "Official schedule identifies 12:00–13:00 SGT. Economic anecdote concerns approximately 2013, not 2021 measurement.", "evidence_class": "practitioner_retrospective_anecdote", "is_ai_study": false, "ai_relevance": "baseline/mechanism: learning, exploit conversion and maintenance; no measured AI effect", "stage": [ "manual_vulnerability_discovery", "exploit_strategy_development", "repeat_exploit_development", "maintenance" ], "sample": { "population": "Chrome RCE work with a new junior employee, approximately 2013", "researcher_count": 1, "independent_replications": null, "sample_source_locator": "printed slide 20 / PDF page 21" }, "observations": [ { "type": "discovery_episode", "duration": 6, "duration_unit": "months", "bugs_found": 0, "phase": "initial", "evidence_class": "retrospective_anecdote", "source_locator": "printed slide 20 / PDF page 21", "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf" }, { "type": "discovery_episode", "duration": 2, "duration_unit": "weeks", "bugs_found": 4, "phase": "subsequent", "evidence_class": "retrospective_anecdote", "source_locator": "printed slide 20 / PDF page 21", "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf" }, { "type": "development_duration", "value": 2, "unit": "months", "denominator": "initial exploit strategy", "evidence_class": "retrospective_anecdote", "source_locator": "printed slide 20 / PDF page 21", "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf" }, { "type": "development_duration", "value": 1, "unit": "week per subsequent exploit", "evidence_class": "retrospective_anecdote", "source_locator": "printed slide 20 / PDF page 21", "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf" }, { "type": "practitioner_fraction_estimate", "value": 50, "unit": "percent", "approximate": true, "denominator": "speaker's discoveries over the preceding several years judged unexploitable", "evidence_class": "practitioner_estimate", "source_locator": "printed slide 21 / PDF page 22", "sample_size": null, "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf" } ], "cost_scope": { "included": [ "recalled discovery and development durations" ], "denominator": "one learning episode; separate exploitability estimate", "failed_attempt_spend": "initial no-finding interval disclosed; money unreported", "monetary_cost_usd": null }, "missing_costs": [ "salary and hours worked", "tools and infrastructure", "maintenance bill" ], "caveats": [ "Elapsed durations are not measured person-hours.", "Do not convert these episodes into a representative productivity ratio or combine the separate exploitability estimate with the four-bug anecdote." ], "source_url": "https://conference.hitb.org/files/hitbsecconf2021sin/materials/KEYNOTE%201%20-%20Security%20Technology%20Arms%20Race%20-%20Mark%20Dowd.pdf", "source_locator": "printed slides 18–25 / PDF pages 19–26", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official event schedule", "url": "https://archive.conference.hitb.org/hitbsecconf2021sin/speakers/mark-dowd/", "archive_url": null, "archive_note": "Organizer's archived site, not a verified third-party capture." } ], "verification": "Official slide text and event schedule rechecked on 2026-10-17.", "headline": "One recalled learning episode: 6 months with no bugs, then 4 bugs in 2 weeks.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC08", "title": "Fuzzing: On the Exponential Cost of Vulnerability Discovery", "speakers": [ "Marcel Böhme", "Brandon Falk" ], "event": "ESEC/FSE 2020", "event_date": null, "event_date_range": [ "2020-11-08", "2020-11-13" ], "date_precision": "event-range", "source_version_note": "2020 conference paper and programme; a 2022 presentation repeats this work and is not an independent study.", "evidence_class": "peer_reviewed_empirical_compute_scaling", "is_ai_study": false, "ai_relevance": "baseline/mechanism: fuzzing capacity and discovery-frontier diminishing returns; not an AI scaling measurement", "stage": [ "fuzzing_benchmark_bug_discovery", "coverage_exploration" ], "sample": { "population": "OSS-Fuzz, Fuzzer Test Suite and open-source program cohorts", "unique_program_total": null, "note": "Cohort sizes are not asserted to be disjoint; no summed unique total." }, "observations": [ { "type": "study_compute_bound", "operator": ">", "value": 4, "unit": "CPU-years", "evidence_class": "reported_empirical_resource_usage", "source_locator": "abstract and section 2.3 / PDF pages 1–2", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "benchmark_configuration", "benchmark": "OSS-Fuzz", "programs": 263, "campaign_hours": 6, "repetitions": 4, "evidence_class": "empirical_design", "source_locator": "section 2.3 / PDF page 2", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "benchmark_configuration", "benchmark": "Fuzzer Test Suite", "programs": 25, "campaign_hours": 6, "repetitions": 20, "evidence_class": "empirical_design", "source_locator": "section 2.3 / PDF page 2", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "benchmark_configuration", "benchmark": "Open-Source", "programs": 6, "repetitions": 10, "evidence_class": "empirical_design", "source_locator": "section 2.3 / PDF page 2", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "scaling_fit", "benchmark": "Fuzzer Test Suite", "analysis_window_minutes": 1, "r_squared_percent": 98.56, "predictor": "logarithm of input-rate machine multiplier", "outcome": "additional known benchmark vulnerabilities found", "evidence_class": "empirical_data_rescaling", "source_locator": "Figure 3(a) / PDF page 4", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "scaling_fit", "benchmark": "Fuzzer Test Suite", "analysis_window_minutes": 8, "r_squared_percent": 97.26, "predictor": "logarithm of input-rate machine multiplier", "outcome": "additional known benchmark vulnerabilities found", "evidence_class": "empirical_data_rescaling", "source_locator": "Figure 3(a) / PDF page 4", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" }, { "type": "qualitative_scaling_result", "value": "Under the input-rate model, extra frontier discoveries have diminishing returns; reaching the same findings sooner scales differently.", "evidence_class": "empirical_data_rescaling_and_model", "source_locator": "sections 3–4; Figure 1", "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf" } ], "cost_scope": { "included": [ "CPU effort and machine-equivalent input rate" ], "denominator": "benchmark findings or coverage within a fixed analysis window", "failed_attempt_spend": "campaign data include non-discovery intervals; no monetary ledger", "monetary_cost_usd": null }, "missing_costs": [ "research and harness labor", "dollar valuation", "exploitability verification" ], "caveats": [ "Machine multipliers rescale recorded timestamps under zero synchronization overhead; they are not a fleet-spending experiment.", "FTS contains known vulnerabilities; crashes and coverage are separate endpoints.", "Fit statistics are not effect sizes or per-bug prices." ], "source_url": "https://mboehme.github.io/paper/FSE20.EmpiricalLaw.pdf", "source_locator": "sections 2.3–2.4, 3–4; Figure 3(a)", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official 2020 programme", "url": "https://2020.esec-fse.org/room/esecfse-2020-venue-virtual-room-1", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "official later presentation of the same paper", "url": "https://2022.esec-fse.org/details/fse-2022-esec-fse-2020/23/Fuzzing-On-the-Exponential-Cost-of-Vulnerability-Discovery", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Author PDF abstract, methods and Figure 3 text rechecked on 2026-10-17; graph point coordinates not digitized.", "headline": "More than 4 CPU-years of fuzzing experiments; compute scaling with diminishing discovery returns.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC09", "title": "The Wolves of Vuln Street: The 1st System Dynamics Model of the 0day Market", "speakers": [ "Katie Moussouris", "Michael Siegel" ], "event": "RSA Conference 2015", "event_date": "2015-04-21", "event_date_range": [ "2015-04-21", "2015-04-21" ], "date_precision": "day", "source_version_note": "MIT research index identifies RSA Conference, Moscone Center, 21 April 2015.", "evidence_class": "system_dynamics_simulation", "is_ai_study": false, "ai_relevance": "baseline/mechanism: discovery capacity, shared techniques and incentives; no measured AI effect", "stage": [ "modeled_discovery", "modeled_rediscovery", "patching_capacity" ], "sample": { "population": "Modeled fixed codebase with heterogeneous vulnerabilities and shared techniques", "empirical_target_count": null }, "observations": [ { "type": "simulation_output", "scenario": "young software", "value": 9, "unit": "percent discovery overlap", "evidence_class": "simulation", "source_locator": "slide 25", "source_url": "https://cams.mit.edu/wp-content/uploads/2017/12/The-Wolves-of-Vuln-Street-The-1st-System-Dynamics-Model-of-the-0day-Market.pdf" }, { "type": "simulation_output", "scenario": "hardened software", "value": 0.8, "unit": "percent discovery overlap", "evidence_class": "simulation", "source_locator": "slide 26", "source_url": "https://cams.mit.edu/wp-content/uploads/2017/12/The-Wolves-of-Vuln-Street-The-1st-System-Dynamics-Model-of-the-0day-Market.pdf" }, { "type": "qualitative_model_mechanism", "value": "Discovery capacity, offensive inventories and patching interact; reusable techniques and tooling incentives affect the flows.", "evidence_class": "conceptual_model", "source_locator": "slides 13–24, 30, 32", "source_url": "https://cams.mit.edu/wp-content/uploads/2017/12/The-Wolves-of-Vuln-Street-The-1st-System-Dynamics-Model-of-the-0day-Market.pdf" } ], "cost_scope": { "included": [ "modeled discovery overlap and capacity mechanisms" ], "denominator": "authors' scenario-specific overlap measure; normalization not reconstructed", "failed_attempt_spend": "not measured", "monetary_cost_usd": null }, "missing_costs": [ "observed researcher expenditure", "calibrated per-finding expense" ], "caveats": [ "The 9% and 0.8% values depend on model parameters; they are not observed rediscovery rates.", "Do not treat their ratio as a causal hardening effect or AI effect." ], "source_url": "https://cams.mit.edu/wp-content/uploads/2017/12/The-Wolves-of-Vuln-Street-The-1st-System-Dynamics-Model-of-the-0day-Market.pdf", "source_locator": "slides 13–26, 30, 32", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "MIT presentation and event index", "url": "https://cams.mit.edu/search-demo/", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Previously downloaded MIT-hosted PDF reread at slides 24–26; MIT event attribution rechecked on 2026-10-17.", "headline": "9% and 0.8% discovery overlap in two model scenarios; simulated, not observed.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC10", "title": "An Empirical Study of Vulnerability Rewards Programs", "speakers": [ "Matthew Finifter", "Devdatta Akhawe", "David Wagner" ], "event": "22nd USENIX Security Symposium", "event_date": null, "event_date_range": [ "2013-08-14", "2013-08-16" ], "date_precision": "event-range", "source_version_note": "2013 proceedings; exact presentation day unverified. Totals describe the historical study period, not current or annual spending.", "evidence_class": "peer_reviewed_vendor_payout_analysis", "is_ai_study": false, "ai_relevance": "baseline/mechanism: vendor acquisition spending differs from researcher production cost; no measured AI effect", "stage": [ "vendor_bounty_payment" ], "sample": { "population": "Chrome and Firefox vulnerability-reward programs", "data_period": "Approximately three years preceding the 2013 paper; exact common start/end dates not established", "exact_period_start": null, "exact_period_end": null }, "observations": [ { "type": "vendor_payout_total", "program": "Chrome", "value": 579605, "unit": "nominal USD", "approximate": false, "denominator": "studied rewards, including development releases", "evidence_class": "observed_vendor_reward_metadata", "source_locator": "section 4.1.5 / printed page 280 / PDF page 9", "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf" }, { "type": "estimated_vendor_payout_total", "program": "Firefox", "value": 570000, "unit": "nominal USD", "approximate": true, "calculation": "190 rewards * assumed 3000 USD", "denominator": "studied rewards, including development releases", "evidence_class": "vendor_payment_estimate", "source_locator": "sections 3.2–3.4 / PDF page 5; section 4.1.5 / PDF page 9", "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf" }, { "type": "reported_count", "program": "Chrome", "value": 501, "unit": "rewarded bugs", "evidence_class": "observed_vendor_reward_metadata", "source_locator": "Table 1 / PDF page 5", "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf" }, { "type": "reported_count", "program": "Firefox", "value": 190, "unit": "rewarded bugs", "evidence_class": "observed_vendor_reward_metadata", "source_locator": "Table 1 / PDF page 5", "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf" }, { "type": "reported_payment_assumption_error_bound", "program": "Firefox", "operator": "<=", "value": 8, "unit": "percent", "evidence_class": "author_estimated_uncertainty", "source_locator": "section 3.4 / PDF page 5", "caveat": "Authors knew 8% of rewards were below the assumed amount; precise lower payments were unavailable.", "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf" } ], "cost_scope": { "included": [ "vendor bounty payments only" ], "denominator": "historical program totals", "failed_attempt_spend": "unpaid research attempts outside vendor payout ledger" }, "missing_costs": [ "program administration", "researcher labor and failed searches", "tools and infrastructure" ], "caveats": [ "Pwnium exploit awards excluded.", "Firefox total is estimated, not an exact payment ledger.", "Do not interpret payout divided by rewarded bugs as discovery production cost." ], "source_url": "https://www.usenix.org/system/files/conference/usenixsecurity13/sec13-paper_finifter.pdf", "source_locator": "sections 3.1–3.4, Table 1, section 4.1.5", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official presentation and recording", "url": "https://www.usenix.org/conference/usenixsecurity13/technical-sessions/presentation/finifter", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Official paper methods, counts, monetary totals and Firefox estimation caveat rechecked on 2026-10-17.", "headline": "Chrome $579,605 in reward-label-derived payouts; Firefox $570,000 estimated over the historical study window.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC11", "title": "You've Got Your Nice List of Bugs, Now What? Vulnerability Discovery and Management Processes in the Wild", "speakers": [ "Noura Alomar", "Primal Wijesekera", "Edward Qiu", "Serge Egelman" ], "event": "SOUPS 2020", "event_date": null, "event_date_range": null, "event_month": "2020-08", "date_precision": "month", "source_version_note": "August 2020 retained because the proceedings cover and internal conference date spans differ; interviews occurred April–June 2019.", "evidence_class": "peer_reviewed_qualitative_interviews", "is_ai_study": false, "ai_relevance": "baseline/mechanism: staffing, verification and remediation constraints; no measured AI effect", "stage": [ "vulnerability_discovery_management", "report_validation", "triage", "remediation" ], "sample": { "population": "Technical and managerial security practitioners", "data_period": [ "2019-04", "2019-06" ], "representative_probability_sample": false }, "observations": [ { "type": "interview_sample_count", "value": 53, "unit": "included participants", "composition": { "pilot_participants": 5, "subsequently_recruited_participants": 48 }, "evidence_class": "qualitative_study_design", "source_locator": "section 3 / PDF pages 3–4", "source_url": "https://www.usenix.org/system/files/soups2020-alomar.pdf" }, { "type": "qualitative_workflow_constraint", "value": "Receiving findings creates verification, impact assessment and remediation work; staffing, trust, scope and funding constrain these processes.", "evidence_class": "qualitative_interview_finding", "source_locator": "sections 6.1–6.2, 8–10", "source_url": "https://www.usenix.org/system/files/soups2020-alomar.pdf" } ], "cost_scope": { "included": [ "qualitative organizational constraints" ], "denominator": "interview themes, not measured report-processing units", "failed_attempt_spend": "not quantified", "monetary_cost_usd": null, "labor_hours_per_finding": null }, "missing_costs": [ "measured labor per report", "dollar expenditure", "queue throughput" ], "caveats": [ "Sample count is study context, not a productivity outcome.", "Qualitative interviews do not establish a numerical bottleneck shift or AI-driven causal trend." ], "source_url": "https://www.usenix.org/system/files/soups2020-alomar.pdf", "source_locator": "section 3, sections 6.1–6.2, 8–10", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official presentation and recording", "url": "https://www.usenix.org/conference/soups2020/presentation/alomar", "archive_url": null, "archive_note": "Matching snapshot not verified." }, { "role": "official slides", "url": "https://www.usenix.org/system/files/soups2020-paper158-slides-alomar.pdf", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Official paper methods and organizational findings rechecked on 2026-10-17.", "headline": "53 practitioner interviews describe validation and remediation work; no measured unit cost.", "source_type": "conference", "comparison_design": "historical_baseline_or_mechanism" }, { "id": "DC12", "title": "Before the CVE: Mining Open-Source Commits for Negative-Day Vulnerabilities", "speakers": [ "Dmitrijs Trizna" ], "event": "[un]prompted.au, September 2026; adapted author article", "event_date": "2026-09-19", "publication_date": "2026-10-02", "date_precision": "day", "evidence_class": "first_party_conference_benchmark", "is_ai_study": true, "stage": [ "security_patch_classification" ], "source_url": "https://aisle.com/blog/before-the-cve-mining-open-source-commits-for-negative-day-vulnerabilities", "source_locator": "Evaluation by Time Travel; Do tools help?; Figures 6–7", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "official schedule; September 19, 15:30 AEST", "url": "https://unprompted.au/schedule" } ], "verification": "Primary account read on 2026-10-17; reported experiments not independently reproduced.", "event_date_range": [ "2026-09-18", "2026-09-19" ], "source_version_note": "Article explicitly attributes the work to the September talk; observations transcribed from the October 2 adaptation.", "ai_relevance": "Matched model/harness comparisons on known security patches; not novel vulnerability discovery.", "comparison_design": "same_corpus_different_models_and_harnesses", "sample": { "commits": 323, "windows": 40, "models": 21, "libraries_approx": 40 }, "observations": [ { "type": "matched_model_comparison", "model": "GPT 5.6 Luna", "cost_usd_per_commit": 0.001, "recall_percent": 90, "approximate_cost": true, "source_locator": "Evaluation by Time Travel; Do tools help?; Figures 6–7", "source_url": "https://aisle.com/blog/before-the-cve-mining-open-source-commits-for-negative-day-vulnerabilities" }, { "type": "matched_model_comparison", "model": "DeepSeek 4.1", "cost_usd_per_commit": 0.002, "recall_percent": 92, "approximate_cost": true, "source_locator": "Evaluation by Time Travel; Do tools help?; Figures 6–7", "source_url": "https://aisle.com/blog/before-the-cve-mining-open-source-commits-for-negative-day-vulnerabilities" }, { "type": "matched_model_comparison", "model": "Sonnet 5", "cost_usd_per_commit": 0.013, "recall_percent": 95, "approximate_cost": true, "source_locator": "Evaluation by Time Travel; Do tools help?; Figures 6–7", "source_url": "https://aisle.com/blog/before-the-cve-mining-open-source-commits-for-negative-day-vulnerabilities" }, { "type": "matched_harness_comparison", "model": "GPT 5.6 Luna", "single": { "cost_usd_per_commit": 0.001, "recall_percent": 90 }, "agent": { "cost_usd_per_commit": 0.0057, "recall_percent": 95 }, "multi_agent": { "recall_percent": 82 }, "approximate_costs": true, "source_locator": "Evaluation by Time Travel; Do tools help?; Figures 6–7", "source_url": "https://aisle.com/blog/before-the-cve-mining-open-source-commits-for-negative-day-vulnerabilities" } ], "cost_scope": { "included": [ "list-price API usage without cache discounts" ], "denominator": "analyzed commit", "failed_attempt_spend": "all classified commits; separate refusal accounting incomplete" }, "missing_costs": [ "engineering", "manual adjudication", "infrastructure" ], "caveats": [ "Private retrospective benchmark; precision needs a larger clean sample.", "Surrounding commits labeled negative may include other security fixes.", "Costs per commit are not costs per discovered vulnerability." ], "headline": "Same patch corpus and model: single-call 90% recall at about $0.001/commit; agent 95% at $0.0057.", "source_type": "conference" }, { "id": "DC13", "title": "Hardening Firefox with Anthropic’s Red Team", "speakers": [ "Brian Grinstead", "Christian Holler" ], "event": null, "event_date": null, "publication_date": "2026-03-06", "date_precision": "day", "evidence_class": "maintainer_corroborated_discovery_report", "is_ai_study": true, "stage": [ "novel_vulnerability_discovery", "maintainer_validation" ], "source_url": "https://blog.mozilla.org/en/firefox/hardening-firefox-anthropic-red-team/", "source_locator": "An emerging technique, pressure-tested by Firefox engineers", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [ { "role": "original experiment and two-week discovery window; distinguishes exploit development costs", "url": "https://www.anthropic.com/news/mozilla-firefox-security" } ], "verification": "Primary account read on 2026-10-17; reported experiments not independently reproduced.", "source_version_note": "Mozilla confirmation and same-day Anthropic methods account; work conducted in February.", "ai_relevance": "Maintainer-confirmed outcomes of AI-assisted discovery; no matched non-AI baseline.", "comparison_design": "uncontrolled_live_target_campaign", "sample": { "project": "Firefox", "model": "Claude Opus 4.6", "reports_submitted": 112, "discovery_window_weeks": 2 }, "observations": [ { "type": "maintainer_confirmed_count", "value": 22, "unit": "security vulnerabilities / CVEs", "source_locator": "An emerging technique, pressure-tested by Firefox engineers", "source_url": "https://blog.mozilla.org/en/firefox/hardening-firefox-anthropic-red-team/" }, { "type": "severity_subset", "value": 14, "unit": "high-severity security vulnerabilities", "denominator": "22 security vulnerabilities", "source_locator": "An emerging technique, pressure-tested by Firefox engineers", "source_url": "https://blog.mozilla.org/en/firefox/hardening-firefox-anthropic-red-team/" }, { "type": "other_bug_count", "value": 90, "unit": "non-security bugs", "source_locator": "An emerging technique, pressure-tested by Firefox engineers", "source_url": "https://blog.mozilla.org/en/firefox/hardening-firefox-anthropic-red-team/" } ], "cost_scope": { "included": [], "denominator": "discovery campaign", "failed_attempt_spend": "not reported", "monetary_cost_usd": null }, "missing_costs": [ "discovery API bill", "human validation", "engineering", "infrastructure" ], "caveats": [ "Anthropic’s separate $4,000 figure covers exploit-development trials, not discovery.", "No measured all-in cost per accepted vulnerability." ], "headline": "Firefox collaboration: 22 security bugs, including 14 high severity, plus 90 other bugs.", "source_type": "maintainer_report", "event_date_range": null }, { "id": "DC14", "title": "The zero-days are numbered", "speakers": [ "Bobby Holley" ], "event": null, "event_date": null, "publication_date": "2026-04-21", "date_precision": "day", "evidence_class": "maintainer_report", "is_ai_study": true, "stage": [ "novel_vulnerability_discovery", "patched_vulnerabilities" ], "source_url": "https://blog.mozilla.org/en/firefox/ai-security-zero-day-vulnerabilities/", "source_locator": "Initial Mythos Preview evaluation and Firefox 150 fixes", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [], "verification": "Primary account read on 2026-10-17; reported experiments not independently reproduced.", "source_version_note": "First-party Mozilla follow-up; observation covers Firefox 150 fixes.", "ai_relevance": "Maintainer reports AI-assisted findings; magnitude of model-caused productivity change is unidentified.", "comparison_design": "uncontrolled_live_target_followup", "sample": { "project": "Firefox", "model": "Claude Mythos Preview", "release": "Firefox 150" }, "observations": [ { "type": "maintainer_reported_count", "value": 271, "unit": "vulnerabilities fixed in Firefox 150 following initial model evaluation", "source_locator": "Initial Mythos Preview evaluation and Firefox 150 fixes", "source_url": "https://blog.mozilla.org/en/firefox/ai-security-zero-day-vulnerabilities/" } ], "cost_scope": { "included": [], "denominator": "initial evaluation findings fixed in release", "failed_attempt_spend": "not reported", "monetary_cost_usd": null }, "missing_costs": [ "API bill", "evaluation duration", "human labor", "coverage and attempt denominator" ], "caveats": [ "Do not turn 22 and 271 into a productivity ratio: model, scope, time and spending differ or are undisclosed.", "This is not a conference presentation or a controlled benchmark." ], "headline": "Firefox 150 fixed 271 vulnerabilities identified during the initial Mythos Preview evaluation.", "source_type": "maintainer_report", "event_date_range": null }, { "id": "DC15", "title": "Vulnerability Discovery and Exploitation Trends in the AI Era", "speakers": [ "Robin Grunewald", "Supriya Mazumdar", "Kelli Vanderlee" ], "event": null, "event_date": null, "publication_date": "2026-09-30", "date_precision": "day", "evidence_class": "industry_observational_report", "is_ai_study": true, "stage": [ "disclosure_population", "AI_attributed_finding_mix" ], "source_url": "https://cloud.google.com/blog/topics/threat-intelligence/vulnerability-discovery-and-exploitation-trends-in-the-ai-era", "source_locator": "Scope & Methodology; CVE Disclosure Doubled in 2026; AI as the Hunter: AI-Assisted Vulnerability Discovery", "archive_url": null, "archive_note": "Matching third-party snapshot not verified.", "supporting_sources": [], "verification": "Primary account read on 2026-10-17; reported experiments not independently reproduced.", "source_version_note": "GTIG analysis window January 2025–August 2026; publication September 30.", "ai_relevance": "Observational association; AI attribution and target-selection biases prevent causal identification.", "comparison_design": "unmatched_observational_cohorts", "sample": { "observation_start": "2025-01-01", "observation_end": "2026-08-31", "AI_cohort_size": null, "non_AI_cohort_size": null }, "observations": [ { "type": "disclosure_count", "period": "2026-01", "value": 5045, "unit": "disclosed vulnerabilities", "source_locator": "Scope & Methodology; CVE Disclosure Doubled in 2026; The Impact of AI on Vulnerability Discovery", "source_url": "https://cloud.google.com/blog/topics/threat-intelligence/vulnerability-discovery-and-exploitation-trends-in-the-ai-era" }, { "type": "disclosure_count", "period": "2026-07", "value": 10477, "unit": "disclosed vulnerabilities", "source_locator": "Scope & Methodology; CVE Disclosure Doubled in 2026; The Impact of AI on Vulnerability Discovery", "source_url": "https://cloud.google.com/blog/topics/threat-intelligence/vulnerability-discovery-and-exploitation-trends-in-the-ai-era" }, { "type": "disclosure_count", "period": "2026-08", "value": 10740, "unit": "disclosed vulnerabilities", "source_locator": "Scope & Methodology; CVE Disclosure Doubled in 2026; The Impact of AI on Vulnerability Discovery", "source_url": "https://cloud.google.com/blog/topics/threat-intelligence/vulnerability-discovery-and-exploitation-trends-in-the-ai-era" }, { "type": "cohort_outcome_share", "outcome": "remote code execution", "AI_attributed_percent": 50, "non_AI_percent": 26, "source_locator": "Scope & Methodology; CVE Disclosure Doubled in 2026; The Impact of AI on Vulnerability Discovery", "source_url": "https://cloud.google.com/blog/topics/threat-intelligence/vulnerability-discovery-and-exploitation-trends-in-the-ai-era" } ], "cost_scope": { "included": [], "denominator": null, "failed_attempt_spend": "not measured" }, "missing_costs": [ "discovery expenditure", "researcher time", "AI attribution completeness" ], "caveats": [ "Disclosure volume includes reporting-policy effects; do not attribute the aggregate increase to AI.", "AI attribution uses vendor ledgers and advisory metadata; cohort sizes are undisclosed.", "RCE proportions may reflect target selection, not a model effect.", "AI-assisted discovery differs from vulnerabilities in AI software." ], "headline": "AI-attributed RCE share 50% versus 26% in the comparison cohort; observational association.", "source_type": "industry_report", "event_date_range": null }, { "id": "DC16", "title": "Harness & Hunt: A Hybrid, Role-Driven Agent Architecture for Autonomous Vulnerability Research", "speakers": [ "Anthony Lai", "Alan Ho", "Ken Wong" ], "event": "[un]prompted.au 2026, Sydney", "event_date": "2026-09-18", "event_date_range": [ "2026-09-18", "2026-09-19" ], "date_precision": "day", "source_version_note": "Current official abstract; exact revision date unverified. No full slides or paper inspected.", "evidence_class": "first_party_conference_abstract", "stage": [ "candidate_discovery", "code_level_adjudication" ], "sample": { "study_months": 3, "models": 2, "findings_adjudicated": 630, "unique_confirmed_vulnerabilities": null, "model_names": null }, "observations": [ { "type": "reported_count", "value": 630, "unit": "candidate findings", "denominator": "study across heterogeneous targets", "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "source_url": "https://unprompted.au/schedule" }, { "type": "reported_outcome_funnel", "cohort": "more expensive frontier model on the static target", "findings": 285, "false_positives": 99, "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "source_url": "https://unprompted.au/schedule" }, { "type": "reported_outcome_funnel", "cohort": "cheaper model on the same static target", "findings": 169, "false_positives": 39, "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "source_url": "https://unprompted.au/schedule" }, { "type": "derived_precision", "cohort": "more expensive model", "value": 65.2632, "unit": "percent", "calculation": "(285-99)/285*100", "denominator": "candidate findings; not recall or unique CVEs", "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "source_url": "https://unprompted.au/schedule" }, { "type": "derived_precision", "cohort": "cheaper model", "value": 76.9231, "unit": "percent", "calculation": "(169-39)/169*100", "denominator": "candidate findings; not recall or unique CVEs", "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "source_url": "https://unprompted.au/schedule" } ], "cost_scope": { "included": [], "denominator": "candidate-level verification", "failed_attempt_spend": "undisclosed", "monetary_cost": null }, "missing_costs": [ "model usage", "manual review hours", "engineering", "failed runs" ], "ai_evidence": { "comparison": "Two models on one shared static target; other methodology varies across the study.", "human_baseline": false, "causal_ai_cost_reduction": false }, "caveats": [ "Human code review remains necessary. Automated judging and pipeline failures can distort apparent results.", "No reported dollar ratio; fewer false positives do not establish greater recall." ], "source_url": "https://unprompted.au/schedule", "source_locator": "Day 1, 15:45–16:15 AEST; Harness & Hunt abstract", "archive_url": null, "archive_note": "Matching snapshot not verified.", "supporting_sources": [], "verification": "Official non-www schedule read on 2026-10-17; findings and false-positive denominators checked directly. Earlier www rendering differed.", "headline": "630 findings adjudicated; one static target yielded 285 findings / 99 false positives versus 169 / 39.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC17", "title": "One Percent of the Tokens, All of the Strategy: LLM-Assisted Vulnerability Discovery in IoT and Embedded Firmware", "speakers": [ "Ta-Lun Yen" ], "event": "Black Hat USA 2026", "event_date": "2026-08-06", "event_date_range": [ "2026-08-05", "2026-08-06" ], "date_precision": "day", "source_version_note": "REV-01 slide file; exact revision date unknown. Use deck outcomes rather than mixing abstract totals.", "evidence_class": "first_party_conference_experiment", "stage": [ "hardware_and_firmware_discovery", "report_triage", "prepared_corpus_chain_identification" ], "sample": { "funnel_target": "Device B", "candidate_findings": 156, "bug_reports": 50, "viable_reports": 25, "critical_reports": 2, "vendor_accepted_count": null, "ablation_runs_per_condition": 3 }, "observations": [ { "type": "reported_outcome_funnel", "candidate_findings": 156, "bug_reports": 50, "viable_reports": 25, "critical_reports": 2, "denominator": "Device B engagement", "source_locator": "PDF page 18", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/USA-26-Yen-LLM-Assisted-Vulnerability-REV-01.pdf" }, { "type": "reported_duration", "value": 3, "unit": "engagement days", "denominator": "Device B", "source_locator": "PDF page 18", "caveat": "Not measured human working hours or pure machine runtime.", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/USA-26-Yen-LLM-Assisted-Vulnerability-REV-01.pdf" }, { "type": "within_workflow_ablation", "baseline_successes": 0, "baseline_attempts": 3, "intervention_successes": 3, "intervention_attempts": 3, "intervention": "context reseeding plus capability graph", "denominator": "identifying a complete chain from the same corpus", "source_locator": "PDF page 42", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/USA-26-Yen-LLM-Assisted-Vulnerability-REV-01.pdf" }, { "type": "reported_retention", "baseline_percent": 48, "intervention_percent": 92, "compactions": 33, "source_locator": "PDF page 42", "caveat": "Retention scoring protocol not established here.", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/USA-26-Yen-LLM-Assisted-Vulnerability-REV-01.pdf" } ], "cost_scope": { "included": [], "denominator": "engagement and small workflow ablation", "failed_attempt_spend": "undisclosed", "monetary_cost": null }, "missing_costs": [ "tokens", "researcher labor", "hardware", "tool engineering", "reporting", "failed attempts" ], "ai_evidence": { "comparison": "Human-guided AI fieldwork plus same-corpus workflow ablation.", "human_baseline": false, "causal_ai_cost_reduction": false }, "caveats": [ "Viable reports are not independently verified vendor acceptances or CVEs.", "The ablation measures chain identification, not new discovery or executed chain reliability.", "The title's one-percent phrase is not a measured 99% cost reduction." ], "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/USA-26-Yen-LLM-Assisted-Vulnerability-REV-01.pdf", "source_locator": "PDF pages 18 and 42", "archive_url": null, "archive_note": "Matching snapshot not verified.", "supporting_sources": [ { "role": "official session; August 6, 11:05–11:45 Pacific", "url": "https://blackhat.com/us-26/briefings/schedule/#one-percent-of-the-tokens-all-of-the-strategy-llm-assisted-vulnerability-discovery-in-iot-and-embedded-firmware-53075", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Official programme and 57-page deck read; PDF pages 18 and 42 visually inspected on 2026-10-17. Page 41 is an incomplete animation build; page 42 displays the results.", "headline": "Device B: 156 findings → 50 reports → 25 viable reports in a three-day engagement.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC18", "title": "Cost-Effective, Private, Frontier-Grade: AI Agent Exploitation with a Fine-Tuned OSS Model", "speakers": [ "Bar Lanyado", "Eliya Cohen" ], "event": "Black Hat USA 2026", "event_date": "2026-08-06", "event_date_range": [ "2026-08-05", "2026-08-06" ], "date_precision": "day", "source_version_note": "AgentBreaker deck has updated results versus the official abstract; exact revision date unknown. Numbers below use slides.", "evidence_class": "first_party_conference_experiment", "stage": [ "ai_agent_security_benchmark", "prompt_injection_validation" ], "sample": { "victim_agents": 30, "scenarios": 15, "security_levels": 2, "total_attack_attempts": null, "total_vulnerable_tools": null }, "observations": [ { "type": "reported_average_scan_cost", "cohort": "30B open model", "value": 0.007, "unit": "USD per scan", "source_locator": "PDF page 5", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_average_scan_cost", "cohort": "frontier models", "value": 0.69, "unit": "USD per scan", "source_locator": "PDF page 5", "caveat": "Averaging weights and detailed billing boundary not disclosed.", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_benchmark_rate", "metric": "ESR", "model": "AgentBreaker", "value": 66.1, "unit": "percent", "denominator": "verified exploited tools / vulnerable tools", "source_locator": "PDF pages 8, 32, 58", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_benchmark_rate", "metric": "ESR", "model": "Nano base", "value": 52.4, "unit": "percent", "denominator": "verified exploited tools / vulnerable tools", "source_locator": "PDF pages 8, 32, 58", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_benchmark_rate", "metric": "ESR", "model": "Opus 4.5", "value": 68.2, "unit": "percent", "denominator": "verified exploited tools / vulnerable tools", "source_locator": "PDF pages 8, 32, 58", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_relative_cost", "value": 1.4, "unit": "times base Nano cost per agent", "cohort": "AgentBreaker", "source_locator": "PDF page 8", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" }, { "type": "reported_training_duration", "value": 1, "unit": "hour on a DGX", "denominator": "LoRA fine-tuning stage", "source_locator": "PDF page 51", "caveat": "Hardware price, data-generation expense and earlier training trials excluded from this duration.", "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf" } ], "cost_scope": { "included": [ "reported average scan execution cost" ], "denominator": "scan; not accepted software vulnerability", "failed_attempt_spend": "not separately itemized", "capital_cost_accounting": null }, "missing_costs": [ "synthetic data generation", "fine-tuning expense", "engineering", "evaluation labor", "GPU utilization and amortization" ], "ai_evidence": { "comparison": "Model variants evaluated on a shared agent-security testbed; narrow task specialization.", "human_baseline": false, "causal_ai_cost_reduction": false }, "caveats": [ "Adjacent AI-agent security task, not conventional zero-day discovery.", "ESR is tool-level coverage, not attack-attempt success or percentage of agents compromised.", "Dollar averages do not establish all-in costs. Abstract rates and slide headline cost ranges differ; no blended multiplier used." ], "source_url": "https://i.blackhat.com/BH-USA-26/Presentations/AgentBreaker-BlackHat.pdf", "source_locator": "PDF pages 5, 8, 32, 51, 58–59", "archive_url": null, "archive_note": "Matching snapshot not verified.", "supporting_sources": [ { "role": "official session; August 6, 10:15–10:45 Pacific", "url": "https://blackhat.com/us-26/briefings/schedule/#cost-effective-private-frontier-grade-ai-agent-exploitation-with-a-fine-tuned-oss-model-52759", "archive_url": null, "archive_note": "Matching snapshot not verified." } ], "verification": "Official programme and 61-page deck read; cost, metric-definition, training and results slides visually inspected on 2026-10-17.", "headline": "Agent-security testbed: $0.007 versus $0.69 average scan cost; training and full hardware cost excluded.", "is_ai_study": true, "source_type": "conference", "comparison_design": "see_observation_specific_conditions", "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC19", "title": "Where Do the Tokens Go? Understanding and Reducing Costs in LLM Agents for Vulnerability Discovery", "speakers": [ "Li Lu", "Yanjie Zhao", "Hongjie Chen", "Haoyu Wang" ], "event": "arXiv preprint", "event_date": null, "publication_date": "2026-10-08", "date_precision": "day", "source_version_note": "v1. PDF conference-header date is an unfilled template, not event evidence.", "evidence_class": "preprint_empirical_study", "stage": [ "harness_bounded_benchmark_crash_discovery", "known_vulnerability_reproduction" ], "sample": { "evaluation_tasks": 20, "runs_per_configuration_task": 1, "separate_diagnostic_traces": 200 }, "observations": [ { "type": "matched_configuration_comparison", "agent": "Codex", "model": "GPT-5.5", "baseline": { "spend_usd": 132.65, "summed_elapsed_seconds": 50968, "success_percent": 85, "recall_percent": 65 }, "AVRI": { "spend_usd": 108.75, "summed_elapsed_seconds": 33313, "success_percent": 85, "recall_percent": 75 }, "reported_spend_reduction_percent": 18.0, "source_locator": "Table 3, PDF page 12", "source_url": "https://arxiv.org/html/2610.11602v1" }, { "type": "matched_configuration_comparison", "agent": "OpenCode", "model": "DeepSeek-V4.1-Flash", "baseline": { "spend_usd": 4.38, "summed_elapsed_seconds": 25324, "success_percent": 100, "recall_percent": 90 }, "AVRI": { "spend_usd": 3.34, "summed_elapsed_seconds": 24017, "success_percent": 100, "recall_percent": 90 }, "reported_spend_reduction_percent": 23.7, "source_locator": "Table 3, PDF page 12", "source_url": "https://arxiv.org/html/2610.11602v1" } ], "cost_scope": { "included": [ "main and auxiliary model billing" ], "denominator": "all 20 selected runs per configuration", "failed_attempt_spend": "included" }, "missing_costs": [ "engineering", "harness preparation", "hardware", "human labor" ], "caveats": [ "Matched tasks and within-agent model; no repeat-run uncertainty estimate.", "Success means nonzero, non-timeout vulnerable-build exit; recall also requires fixed-build success.", "Known-vulnerable repositories and supplied harness; not unrestricted novel discovery." ], "source_url": "https://arxiv.org/html/2610.11602v1", "source_locator": "sections 2.1, 2.4, 5-6; Table 3", "archive_url": null, "archive_note": "No third-party snapshot verified; versioned primary URL retained.", "supporting_sources": [ { "role": "dated metadata", "url": "https://arxiv.org/abs/2610.11602" } ], "verification": "Methods read; PDF Table 3 visually checked, 2026-10-17. Experiments not independently reproduced.", "headline": "AVRI: matched 20-task Codex campaign cost $132.65 → $108.75; crash success stayed at 85%.", "is_ai_study": true, "source_type": "preprint", "comparison_design": "see_observation_specific_conditions", "event_date_range": null, "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC20", "title": "Revelio: Cost-Efficient Agentic Memory Safety Vulnerability Detection For Repository-Scale Codebases", "speakers": [ "Yiwei Hou", "Hao Wang", "Muxi Lyu", "Marius Momeu", "Eric Nguyen", "Taige Yang", "Koushik Sen", "Dawn Song", "David Wagner" ], "event": "arXiv preprint", "event_date": null, "publication_date": "2026-06-20", "date_precision": "day", "source_version_note": "v1; discovery reports occurred in April 2026.", "evidence_class": "preprint_empirical_study", "stage": [ "previously_unknown_memory_safety_discovery", "human_validation" ], "sample": { "oss_fuzz_projects": 7, "author_validated_findings": 19, "maintainer_fixed_findings_at_reporting": 10, "cve_assigned_or_requested": 7, "excluded_null_dereference_bugs": 16 }, "observations": [ { "type": "reported_spend", "value": 300, "unit": "USD API", "approximate": true, "denominator": "seven-project discovery campaign", "source_locator": "section V-A; Tables III, V, VII; cost accounting in section V", "source_url": "https://arxiv.org/html/2606.22263v1" }, { "type": "reported_ratio", "value": 16, "unit": "USD API per author-validated finding", "approximate": true, "source_locator": "section V-A; Tables III, V, VII; cost accounting in section V", "source_url": "https://arxiv.org/html/2606.22263v1" }, { "type": "reported_duration", "value": 65, "unit": "minutes median runtime per project", "approximate": true, "source_locator": "section V-A; Tables III, V, VII; cost accounting in section V", "source_url": "https://arxiv.org/html/2606.22263v1" } ], "cost_scope": { "included": [ "all model calls, including hypothesis generation and confirmation" ], "denominator": "19 manually validated findings", "failed_attempt_spend": "campaign-wide model calls; no failure-specific ledger" }, "missing_costs": [ "manual triage", "engineering", "hardware", "disclosure" ], "access_and_supervision": "Haiku 4.5 and Sonnet 4.6; existing OSS-Fuzz harnesses; manual validation before disclosure.", "caveats": [ "Novel-discovery campaign has no matched human or non-AI baseline.", "Author validation is broader than maintainer acceptance; zero-false-positive wording must retain the exclusions.", "Separate CyberGym comparison supplies the vulnerable file; do not substitute its denominator." ], "source_url": "https://arxiv.org/html/2606.22263v1", "source_locator": "section V-A; Tables III, V, VII; cost accounting in section V", "archive_url": null, "archive_note": "No third-party snapshot verified; versioned primary URL retained.", "supporting_sources": [ { "role": "dated metadata", "url": "https://arxiv.org/abs/2606.22263" } ], "verification": "Methods read; PDF Tables 5 and 7 visually checked, 2026-10-17. No independent replication.", "headline": "Revelio: about $300 API spend for 19 author-validated findings across 7 OSS-Fuzz projects.", "is_ai_study": true, "source_type": "preprint", "comparison_design": "see_observation_specific_conditions", "event_date_range": null, "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC21", "title": "Towards Scalable and Cost-Efficient Vulnerability Detection: A Study on Automatic Query Generation", "speakers": [ "Ivana Clairine Irsan", "Ratnadira Widyasari", "Huihui Huang", "Ting Zhang", "Yue Liu", "Ouh Eng Lieh", "Shar Lwin Khin", "Kang Hong Jin", "David Lo" ], "event": "arXiv preprint", "event_date": null, "publication_date": "2026-09-09", "date_precision": "day", "source_version_note": "v1", "evidence_class": "preprint_empirical_study", "stage": [ "reusable_query_generation", "known_cve_file_detection" ], "sample": { "mining_cves": 80, "evaluation_cves": 112, "cwe_categories": 10 }, "observations": [ { "type": "reported_spend", "model": "Kimi K2.5", "value": 12.99, "unit": "USD", "denominator": "96 compilable CodeQL queries", "source_locator": "Table 3, PDF page 10", "source_url": "https://arxiv.org/html/2609.10412v1" }, { "type": "reported_duration", "value": 24, "unit": "hours", "denominator": "query-generation campaign", "source_locator": "Table 3", "source_url": "https://arxiv.org/html/2609.10412v1" }, { "type": "matched_detection_comparison", "baseline_CodeQL": { "detected": 19, "average_fdr_percent": 89.19, "average_f1_percent": 13.2 }, "generated_queries": { "detected": 50, "average_fdr_percent": 89.92, "average_f1_percent": 24.04 }, "denominator": 112, "source_locator": "Table 4, PDF page 11", "source_url": "https://arxiv.org/html/2609.10412v1" } ], "cost_scope": { "included": [ "reported OpenRouter query-synthesis expense, with syntax refinement" ], "denominator": "query suite, not new discoveries", "failed_attempt_spend": "iterative campaign reported; granular unsuccessful-call accounting unspecified" }, "missing_costs": [ "CodeQL execution valuation", "human review", "engineering" ], "caveats": [ "Known-CVE benchmark; detection means hitting a ground-truth file, not reproducing a vulnerability.", "19 to 50 is approximately 163% increase, not the paper's 263% increase.", "Separate $17/44-hour scan covers one selected repository; no full matched direct-scanning comparison." ], "source_url": "https://arxiv.org/html/2609.10412v1", "source_locator": "sections 2.2, 4.1-4.5, 5 RQ4; Tables 3-4", "archive_url": null, "archive_note": "No third-party snapshot verified; versioned primary URL retained.", "supporting_sources": [ { "role": "dated metadata", "url": "https://arxiv.org/abs/2609.10412" } ], "verification": "Methods and PDF Tables 3-4 visually checked, 2026-10-17; arithmetic independently checked.", "headline": "$12.99 generated 96 reusable queries; known-CVE file detections 19 → 50 of 112, with high false discovery.", "is_ai_study": true, "source_type": "preprint", "comparison_design": "see_observation_specific_conditions", "event_date_range": null, "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC22", "title": "Drishti: AI-Led Human-Directed Vulnerability Auditing for 5G Cores", "speakers": [ "Sriram Ramachandran", "Levente Csikor", "Dinil Mon Divakaran" ], "event": "arXiv preprint; paper labels 8th Joint Workshop on CPS & IoT Security and Privacy", "event_date": null, "publication_date": "2026-08-31", "date_precision": "day", "source_version_note": "v1; conference occurrence/date not separately verified.", "evidence_class": "preprint_case_study", "stage": [ "candidate_review", "reachability_validation", "patch_review" ], "sample": { "software_projects": 2, "reported_findings": 3, "novel_discoveries": 2, "patch_review_findings": 1 }, "observations": [ { "type": "audit_stage_counts", "audit_cell": "NAS AP1", "raw_matches": 89, "reachable_pre_authentication_IE_types": 8, "representative_cells_runtime_validated": 1, "source_locator": "Appendix A.3, Table 3", "source_url": "https://arxiv.org/html/2608.30112v1", "in_scope_sites": 45, "confirmed_sites": 45 }, { "type": "deferred_review", "audit_cell": "NAS AP8", "raw_matches": 983, "reason": "search too noisy without decoder filter", "source_locator": "Appendix A.3, Table 3", "source_url": "https://arxiv.org/html/2608.30112v1" } ], "cost_scope": { "included": [], "denominator": null, "failed_attempt_spend": "not reported", "time_measurement": "End-to-end operator time not consistently captured." }, "missing_costs": [ "API expense", "operator time", "lab construction", "curation" ], "access_and_supervision": "Claude Opus 4.7/4.8; human-directed audit and impact validation.", "caveats": [ "Confirmed code sites are not separate exploitable vulnerabilities; sites, IE types and validation cells use different units and cannot yield funnel conversion rates.", "No controlled comparison; heterogeneous routes cannot share one false-positive rate.", "Runtime experiment durations measure impact tests, not discovery labor." ], "source_url": "https://arxiv.org/html/2608.30112v1", "source_locator": "sections 3.1, 5; Appendices A.3-A.4", "archive_url": null, "archive_note": "No third-party snapshot verified; versioned primary URL retained.", "supporting_sources": [ { "role": "dated metadata", "url": "https://arxiv.org/abs/2608.30112" } ], "verification": "Primary HTML methods and accounting read, 2026-10-17; no replication.", "headline": "Drishti: 89 raw matches; 45 confirmed code sites; 8 reachable IE types; one representative cell validated. Units change at each step.", "is_ai_study": true, "source_type": "preprint", "comparison_design": "see_observation_specific_conditions", "event_date_range": null, "ai_relevance": "AI workflow evidence; not a matched comparison against human research." }, { "id": "DC23", "title": "Mastermind: Strategy-grounded Learning for Repository-Scale Vulnerability Reproduction", "speakers": [ "Mingzhe Du", "Luu Anh Tuan", "Tianyi Wu", "Renyang Liu", "Zhijiang Guo", "Dong Huang", "See-Kiong Ng" ], "event": "arXiv preprint", "event_date": null, "publication_date": "2026-07-02", "date_precision": "day", "source_version_note": "v1", "evidence_class": "preprint_empirical_study", "stage": [ "known_vulnerability_reproduction" ], "sample": { "training_tasks": 260, "held_out_evaluation_tasks": 200, "maximum_attempts_per_task": 8 }, "observations": [ { "type": "matched_executor_comparison", "model": "GPT-5.5", "baseline_Best_of_8": { "passes": 126, "pass_percent": 63.0, "estimated_payg_equivalent_usd": 3600, "estimated_serial_hours": 160, "estimated_usd_per_pass": 29 }, "Mastermind": { "passes": 169, "pass_percent": 84.5, "estimated_payg_equivalent_usd": 1300, "estimated_serial_hours_range": [ 55, 70 ], "estimated_usd_per_pass": 8 }, "approximate_costs": true, "source_locator": "Tables VI-VII; sections V-D/V-E", "source_url": "https://arxiv.org/html/2607.01764v1" } ], "cost_scope": { "included": [ "coarse pay-as-you-go equivalents for evaluation execution" ], "denominator": "200-task evaluation; strict target-crash and patched-build validation", "failed_attempt_spend": "evaluation estimates cover rollout campaigns; fine-grained failed-start accounting incomplete", "actual_billing": "Executors used subscriptions; dollar values are not actual invoices." }, "missing_costs": [ "planner training", "engineering", "human labor", "hardware" ], "caveats": [ "Matched task set and frozen executor; methods differ in planning, experience sharing and early stopping.", "Known-bug reproduction, not new zero-days.", "Training separately requires 4,160 executor rollouts per 260-task pass, plus planner calls." ], "source_url": "https://arxiv.org/html/2607.01764v1", "source_locator": "sections IV-C/IV-D, V-D/V-E, VIII; Tables VI-VII", "archive_url": null, "archive_note": "No third-party snapshot verified; versioned primary URL retained.", "supporting_sources": [ { "role": "dated metadata", "url": "https://arxiv.org/abs/2607.01764" } ], "verification": "Primary HTML accounting, comparisons and success definitions checked, 2026-10-17; no replication.", "headline": "Mastermind: 126 → 169 successes on 200 known-bug tasks; estimated execution-equivalent cost $3,600 → $1,300.", "is_ai_study": true, "source_type": "preprint", "comparison_design": "see_observation_specific_conditions", "event_date_range": null, "ai_relevance": "AI workflow evidence; not a matched comparison against human research." } ] }