{ "experiment": "sap-llm-benchmark-v1", "target": "SAP SuccessFactors", "generated_at": "2026-03-26T22:33:12.417905", "total_runs": 5, "total_cost": 0.051992, "stack_ranking": [ { "rank": 1, "combo_name": "Production Baseline (DeepSeek+Mistral)", "avg_quality": 64.25, "total_cost": 0.010641, "total_latency_ms": 242942, "models": { "researcher_2": "mistral/mistral-small-latest", "researcher_1": "deepseek/deepseek-chat", "certifier": "openai/gpt-4o-mini", "merger": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "deepseek/deepseek-chat" } }, { "rank": 2, "combo_name": "Llama vs DeepSeek (Together+DeepSeek)", "avg_quality": 61.53, "total_cost": 0.014995, "total_latency_ms": 415707, "models": { "researcher_2": "deepseek/deepseek-chat", "researcher_1": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "certifier": "local_claude/claude-code-desktop", "merger": "openai/gpt-4o-mini", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "mistral/mistral-small-latest" } }, { "rank": 3, "combo_name": "Premium Pair (OpenAI+Together)", "avg_quality": 61.04, "total_cost": 0.009572, "total_latency_ms": 281870, "models": { "researcher_2": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "researcher_1": "openai/gpt-4o-mini", "certifier": "local_claude/claude-code-desktop", "merger": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "openai/gpt-4o-mini" } }, { "rank": 4, "combo_name": "Speed vs Quality (Groq+OpenAI)", "avg_quality": 58.9, "total_cost": 0.003484, "total_latency_ms": 261315, "models": { "researcher_2": "openai/gpt-4o-mini", "researcher_1": "groq/llama-3.3-70b-versatile", "certifier": "local_claude/claude-code-desktop", "merger": "mistral/mistral-small-latest", "synthesizer": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop" } }, { "rank": 5, "combo_name": "Budget Tier (Mistral+Groq)", "avg_quality": 58.13, "total_cost": 0.0133, "total_latency_ms": 250483, "models": { "researcher_2": "groq/llama-3.3-70b-versatile", "researcher_1": "mistral/mistral-small-latest", "certifier": "openai/gpt-4o-mini", "merger": "deepseek/deepseek-chat", "synthesizer": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "synthesizer_claude": "local_claude/claude-code-desktop" } } ], "runs": { "1": { "run_number": 1, "combo_name": "Production Baseline (DeepSeek+Mistral)", "run_id": "run_1_deepseek_mistral", "tasks": [ { "task_type": "b4a_extraction", "task_role": "researcher_2", "provider": "mistral", "model": "mistral-small-latest", "quality_score": 96.0, "cost_usd": 0.00186, "latency_ms": 23808, "input_tokens": 6832, "output_tokens": 3924, "json_valid": true, "error": null }, { "task_type": "b4a_extraction", "task_role": "researcher_1", "provider": "deepseek", "model": "deepseek-chat", "quality_score": 95.33, "cost_usd": 0.005205, "latency_ms": 84678, "input_tokens": 6564, "output_tokens": 3121, "json_valid": true, "error": null }, { "task_type": "b4b_certification", "task_role": "certifier", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 26.92, "cost_usd": 0.001378, "latency_ms": 15829, "input_tokens": 4820, "output_tokens": 1092, "json_valid": true, "error": null, "verdict": "UNKNOWN", "confidence": 0.0 }, { "task_type": "b4b_validation", "task_role": "merger", "provider": "groq", "model": "llama-3.3-70b-versatile", "quality_score": 55.25, "cost_usd": 0.0, "latency_ms": 7974, "input_tokens": 5636, "output_tokens": 2775, "json_valid": true, "error": null }, { "task_type": "b5_narrative", "task_role": "synthesizer_claude", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 61.0, "cost_usd": 0.0, "latency_ms": 65418, "input_tokens": 1952, "output_tokens": 2912, "json_valid": true, "error": null, "fit_score": 68, "cold_call_opener": "SAP SuccessFactors is selling HCM to HR leaders every day, but Workday is winning the narrative war because they don't own the community where those HR leaders actually form their opinions \u2014 I have a way to fix that.", "email_hook": "SAP SuccessFactors has the best HCM product in enterprise, but every year Workday closes deals because they've built a perception of being the 'HR professional's platform' \u2014 HR.com's 1.92M members and category-defining domain are the acquisition that flips that dynamic permanently." }, { "task_type": "b5_narrative", "task_role": "synthesizer", "provider": "deepseek", "model": "deepseek-chat", "quality_score": 51.0, "cost_usd": 0.002198, "latency_ms": 45235, "input_tokens": 1701, "output_tokens": 1581, "json_valid": true, "error": null, "fit_score": 65, "cold_call_opener": "To own the HR profession's digital center of gravity and its high-intent demand signals, you need to acquire the platform where 1.92M practitioners already gather\u2014HR.com.", "email_hook": "Integrating HR.com's 1.92M-member community and behavioral intent data directly into the SuccessFactors platform would create an unassailable ecosystem, transforming how you identify, engage, and retain enterprise HR customers." } ], "total_cost": 0.010641000000000001, "total_latency": 242942, "avg_quality": 64.25, "models_used": { "researcher_2": "mistral/mistral-small-latest", "researcher_1": "deepseek/deepseek-chat", "certifier": "openai/gpt-4o-mini", "merger": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "deepseek/deepseek-chat" } }, "2": { "run_number": 2, "combo_name": "Speed vs Quality (Groq+OpenAI)", "run_id": "run_2_groq_openai", "tasks": [ { "task_type": "b4a_extraction", "task_role": "researcher_2", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 87.0, "cost_usd": 0.001923, "latency_ms": 28747, "input_tokens": 6360, "output_tokens": 1615, "json_valid": true, "error": null }, { "task_type": "b4a_extraction", "task_role": "researcher_1", "provider": "groq", "model": "llama-3.3-70b-versatile", "quality_score": 81.83, "cost_usd": 0.0, "latency_ms": 2997, "input_tokens": 5582, "output_tokens": 805, "json_valid": true, "error": null }, { "task_type": "b4b_certification", "task_role": "certifier", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 36.25, "cost_usd": 0.0, "latency_ms": 140788, "input_tokens": 12899, "output_tokens": 2449, "json_valid": true, "error": null, "verdict": "REJECTED", "confidence": 0.0 }, { "task_type": "b4b_validation", "task_role": "merger", "provider": "mistral", "model": "mistral-small-latest", "quality_score": 41.33, "cost_usd": 0.001561, "latency_ms": 16733, "input_tokens": 4440, "output_tokens": 3724, "json_valid": true, "error": null }, { "task_type": "b5_narrative", "task_role": "synthesizer", "provider": "groq", "model": "llama-3.3-70b-versatile", "quality_score": 51.0, "cost_usd": 0.0, "latency_ms": 3523, "input_tokens": 1712, "output_tokens": 948, "json_valid": true, "error": null, "fit_score": 60, "cold_call_opener": "Given the importance of community and data-driven insights in the HR tech space, acquiring HR.com could be the strategic move that sets SAP SuccessFactors apart from its competitors and addresses its potential challenges in these areas.", "email_hook": "As SAP SuccessFactors continues to evolve its HR technology offerings, integrating a robust community and comprehensive data insights could be the key to unlocking new growth opportunities and enhancing its market position." }, { "task_type": "b5_narrative", "task_role": "synthesizer_claude", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 56.0, "cost_usd": 0.0, "latency_ms": 68527, "input_tokens": 1952, "output_tokens": 2895, "json_valid": true, "error": null, "fit_score": 72, "cold_call_opener": "SAP SuccessFactors is betting its enterprise AI roadmap on Joule, but right now you have no visibility into the 1.92 million HR professionals who are actively forming their HCM shortlists before they ever enter your pipeline \u2014 HR.com has that behavioral data, and Workday is aware it's available.", "email_hook": "SAP SuccessFactors's Joule AI copilot is only as intelligent as the upstream signal feeding it \u2014 and the world's largest registered community of HR professionals, 1.92 million members with full behavioral data on content consumption and vendor intent, is currently available as an acquisition rather than a gap in your data stack." } ], "total_cost": 0.003484, "total_latency": 261315, "avg_quality": 58.901666666666664, "models_used": { "researcher_2": "openai/gpt-4o-mini", "researcher_1": "groq/llama-3.3-70b-versatile", "certifier": "local_claude/claude-code-desktop", "merger": "mistral/mistral-small-latest", "synthesizer": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop" } }, "3": { "run_number": 3, "combo_name": "Llama vs DeepSeek (Together+DeepSeek)", "run_id": "run_3_together_deepseek", "tasks": [ { "task_type": "b4a_extraction", "task_role": "researcher_2", "provider": "deepseek", "model": "deepseek-chat", "quality_score": 95.33, "cost_usd": 0.004839, "latency_ms": 74745, "input_tokens": 6564, "output_tokens": 2788, "json_valid": true, "error": null }, { "task_type": "b4a_extraction", "task_role": "researcher_1", "provider": "together", "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "quality_score": 87.67, "cost_usd": 0.00681, "latency_ms": 17267, "input_tokens": 6498, "output_tokens": 1241, "json_valid": true, "error": null }, { "task_type": "b4b_certification", "task_role": "certifier", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 31.25, "cost_usd": 0.0, "latency_ms": 164430, "input_tokens": 10506, "output_tokens": 2158, "json_valid": true, "error": null, "verdict": "CERTIFIED", "confidence": 0.0 }, { "task_type": "b4b_validation", "task_role": "merger", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 42.92, "cost_usd": 0.00265, "latency_ms": 57256, "input_tokens": 5524, "output_tokens": 3036, "json_valid": true, "error": null }, { "task_type": "b5_narrative", "task_role": "synthesizer_claude", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 61.0, "cost_usd": 0.0, "latency_ms": 90121, "input_tokens": 1952, "output_tokens": 2901, "json_valid": true, "error": null, "fit_score": 62, "cold_call_opener": "Your Intelligent Enterprise strategy is compelling, but Workday is winning the practitioner credibility battle \u2014 I'm calling because we represent an asset that gives SAP SuccessFactors ownership of the 1.92 million HR professionals who decide which HCM platform their company buys next.", "email_hook": "SAP SuccessFactors has the strongest enterprise HCM product in the market, but the practitioner community where HR leaders benchmark peers, earn credentials, and evaluate vendors is currently owned by a third party \u2014 and we're bringing that asset to market." }, { "task_type": "b5_narrative", "task_role": "synthesizer", "provider": "mistral", "model": "mistral-small-latest", "quality_score": 51.0, "cost_usd": 0.000696, "latency_ms": 11888, "input_tokens": 1774, "output_tokens": 1730, "json_valid": true, "error": null, "fit_score": 95, "cold_call_opener": "SAP SuccessFactors has publicly acknowledged the need to move beyond software licensing and own the HR professional\u2019s journey\u2014HR.com\u2019s 1.92M members, behavioral data, and white-label community platform give you the audience and infrastructure to do exactly that, today.", "email_hook": "If SAP SuccessFactors aims to become the central platform for HR professionals\u2019 career growth, HR.com\u2019s 1.92M members, proprietary research, and vendor marketplace are the missing pieces to turn that vision into revenue\u2014without building a single new feature." } ], "total_cost": 0.014995, "total_latency": 415707, "avg_quality": 61.528333333333336, "models_used": { "researcher_2": "deepseek/deepseek-chat", "researcher_1": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "certifier": "local_claude/claude-code-desktop", "merger": "openai/gpt-4o-mini", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "mistral/mistral-small-latest" } }, "4": { "run_number": 4, "combo_name": "Budget Tier (Mistral+Groq)", "run_id": "run_4_mistral_groq", "tasks": [ { "task_type": "b4a_extraction", "task_role": "researcher_2", "provider": "groq", "model": "llama-3.3-70b-versatile", "quality_score": 82.5, "cost_usd": 0.0, "latency_ms": 3476, "input_tokens": 5582, "output_tokens": 892, "json_valid": true, "error": null }, { "task_type": "b4a_extraction", "task_role": "researcher_1", "provider": "mistral", "model": "mistral-small-latest", "quality_score": 94.67, "cost_usd": 0.003666, "latency_ms": 40961, "input_tokens": 13686, "output_tokens": 7659, "json_valid": true, "error": null }, { "task_type": "b4b_certification", "task_role": "certifier", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 21.92, "cost_usd": 0.001683, "latency_ms": 29960, "input_tokens": 5723, "output_tokens": 1374, "json_valid": true, "error": null, "verdict": "UNKNOWN", "confidence": 0.0 }, { "task_type": "b4b_validation", "task_role": "merger", "provider": "deepseek", "model": "deepseek-chat", "quality_score": 47.67, "cost_usd": 0.005499, "latency_ms": 78804, "input_tokens": 5285, "output_tokens": 3702, "json_valid": true, "error": null }, { "task_type": "b5_narrative", "task_role": "synthesizer", "provider": "together", "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "quality_score": 46.0, "cost_usd": 0.002452, "latency_ms": 24014, "input_tokens": 1696, "output_tokens": 1090, "json_valid": true, "error": null, "fit_score": 80, "cold_call_opener": "As a leader in the HR technology space, I believe SAP SuccessFactors needs to enhance its community engagement and customer understanding, which is exactly what HR.com's 1.92M HR Professional Members and behavioral data can provide.", "email_hook": "I've identified a strategic opportunity for SAP SuccessFactors to acquire HR.com, leveraging its unique assets to enhance customer engagement, market presence, and competitive positioning." }, { "task_type": "b5_narrative", "task_role": "synthesizer_claude", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 56.0, "cost_usd": 0.0, "latency_ms": 73268, "input_tokens": 1952, "output_tokens": 3344, "json_valid": true, "error": null, "fit_score": 68, "cold_call_opener": "SAP SuccessFactors has built the most comprehensive HCM suite on the market, but Workday owns more of the CHRO mindshare \u2014 and I have a proposal to give SAP the 1.92 million HR practitioners who decide which platform their company buys next.", "email_hook": "SAP SuccessFactors powers the workforce of thousands of enterprises, but the 1.92 million HR professionals who influence those purchasing decisions go to HR.com \u2014 not SuccessFactors \u2014 every day for accredited learning, peer community, and vendor research, and I believe that should change." } ], "total_cost": 0.0133, "total_latency": 250483, "avg_quality": 58.12666666666667, "models_used": { "researcher_2": "groq/llama-3.3-70b-versatile", "researcher_1": "mistral/mistral-small-latest", "certifier": "openai/gpt-4o-mini", "merger": "deepseek/deepseek-chat", "synthesizer": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "synthesizer_claude": "local_claude/claude-code-desktop" } }, "5": { "run_number": 5, "combo_name": "Premium Pair (OpenAI+Together)", "run_id": "run_5_openai_together", "tasks": [ { "task_type": "b4a_extraction", "task_role": "researcher_2", "provider": "together", "model": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "quality_score": 87.67, "cost_usd": 0.00667, "latency_ms": 20185, "input_tokens": 6498, "output_tokens": 1081, "json_valid": true, "error": null }, { "task_type": "b4a_extraction", "task_role": "researcher_1", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 87.67, "cost_usd": 0.001994, "latency_ms": 29557, "input_tokens": 6360, "output_tokens": 1733, "json_valid": true, "error": null }, { "task_type": "b4b_certification", "task_role": "certifier", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 36.25, "cost_usd": 0.0, "latency_ms": 136903, "input_tokens": 8812, "output_tokens": 2349, "json_valid": true, "error": null, "verdict": "REJECTED", "confidence": 0.0 }, { "task_type": "b4b_validation", "task_role": "merger", "provider": "groq", "model": "llama-3.3-70b-versatile", "quality_score": 42.67, "cost_usd": 0.0, "latency_ms": 6315, "input_tokens": 4600, "output_tokens": 2061, "json_valid": true, "error": null }, { "task_type": "b5_narrative", "task_role": "synthesizer_claude", "provider": "local_claude", "model": "claude-code-desktop", "quality_score": 61.0, "cost_usd": 0.0, "latency_ms": 69609, "input_tokens": 1952, "output_tokens": 2990, "json_valid": true, "error": null, "fit_score": 67, "cold_call_opener": "SAP SuccessFactors sells HCM to every major HR department in the world, but you don't own the community where those HR teams learn, benchmark, and decide which modules to buy next \u2014 and we have an asset that fixes that.", "email_hook": "SAP SuccessFactors is the system of record for HR in thousands of enterprises, but HR.com is the system of influence \u2014 1.92 million HR professionals who make or shape every HCM buying decision \u2014 and I think there's a compelling case for why SAP should own both." }, { "task_type": "b5_narrative", "task_role": "synthesizer", "provider": "openai", "model": "gpt-4o-mini", "quality_score": 51.0, "cost_usd": 0.000908, "latency_ms": 19301, "input_tokens": 1660, "output_tokens": 1098, "json_valid": true, "error": null, "fit_score": 80, "cold_call_opener": "SAP SuccessFactors needs to enhance its audience engagement and data-driven marketing strategies, and HR.com offers the largest community of HR professionals to achieve that.", "email_hook": "To strengthen your market position, consider how HR.com\u2019s 1.92M HR Professional Members can provide the first-party data you need for effective targeting." } ], "total_cost": 0.009572, "total_latency": 281870, "avg_quality": 61.04333333333333, "models_used": { "researcher_2": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "researcher_1": "openai/gpt-4o-mini", "certifier": "local_claude/claude-code-desktop", "merger": "groq/llama-3.3-70b-versatile", "synthesizer_claude": "local_claude/claude-code-desktop", "synthesizer": "openai/gpt-4o-mini" } } }, "model_leaderboard": [ { "model": "together/meta-llama/Llama-3.3-70B-Instruct-Turbo", "appearances": 3, "avg_score": 73.78000000000002, "avg_cost": 0.005310666666666666, "avg_latency": 20488.666666666668, "roles": [ "researcher_2", "synthesizer", "researcher_1" ] }, { "model": "deepseek/deepseek-chat", "appearances": 4, "avg_score": 72.3325, "avg_cost": 0.00443525, "avg_latency": 70865.5, "roles": [ "merger", "researcher_2", "synthesizer", "researcher_1" ] }, { "model": "mistral/mistral-small-latest", "appearances": 4, "avg_score": 70.75, "avg_cost": 0.0019457500000000002, "avg_latency": 23347.5, "roles": [ "merger", "researcher_2", "synthesizer", "researcher_1" ] }, { "model": "groq/llama-3.3-70b-versatile", "appearances": 5, "avg_score": 62.65, "avg_cost": 0.0, "avg_latency": 4857.0, "roles": [ "merger", "researcher_2", "synthesizer", "researcher_1" ] }, { "model": "openai/gpt-4o-mini", "appearances": 6, "avg_score": 52.905, "avg_cost": 0.001756, "avg_latency": 30108.333333333332, "roles": [ "certifier", "researcher_2", "synthesizer", "merger", "researcher_1" ] }, { "model": "local_claude/claude-code-desktop", "appearances": 8, "avg_score": 49.84375, "avg_cost": 0.0, "avg_latency": 101133.0, "roles": [ "certifier", "synthesizer_claude" ] } ] }