r/palladium
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Token Calculator

Browse Source
This commit is contained in:
Robert Helewka
2026-06-10 14:28:16 -04:00
parent 64fb83257d
commit 71b98ee4e4
20 changed files with 9719 additions and 916 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
studies/202512_GenesysCX/ctm-token-calculator/tests/test_benefit_model.py
View File

@@ -8,9 +8,11 @@ from tokencalc.benefit_model import (
calculate_acw_summarization_benefit,
calculate_email_ai_benefit,
calculate_total_benefit,
calculate_va_deflection_benefit,
)
from tokencalc.defaults import CTM_DEFAULT_FEATURE_SCOPES, CTM_DEFAULT_SITES
from tokencalc.inputs import WORKING_SECONDS_PER_YEAR, FeatureScope, SiteInput
from tokencalc.scenarios import BENEFIT_PARAMS
ALL_SITES = [s.site_name for s in CTM_DEFAULT_SITES]
@@ -105,3 +107,133 @@ def test_zero_volume_site_is_safe():
def test_working_seconds_constant():
assert WORKING_SECONDS_PER_YEAR == 2_080 * 3_600
# ── Virtual Agent deflection tests ───────────────────────────────────────────
def test_va_bot_deflection_hand_check():
"""Voice Bot: 10,000 calls/mo × 12 × 35% bot_rate × 300s AHT
× 50% Y1 realization × realization_factor × $0.01/s.
realistic realization_factor = 0.70 × 0.80 × (1 − 0.05) = 0.532
"""
site = _small_site()
df = calculate_va_deflection_benefit(
[site],
FeatureScope("Voice Bot", ["Small"], deflection_target=0.35),
"realistic",
year=1,
params="realistic",
)
completion = BENEFIT_PARAMS["va_completion_rate"]["realistic"]
labour = BENEFIT_PARAMS["va_labour_realization"]["realistic"]
callback = BENEFIT_PARAMS["va_callback_discount"]["realistic"]
real_factor = completion * labour * (1.0 - callback)
expected = (
10_000 * 12 # annual calls
* 0.35 # bot deflection rate
* 300 # AHT seconds
* 0.50 # Y1 scenario realization
* real_factor # completion × labour × (1 − callback)
* 0.01 # labour rate per second
)
assert df["annual_value"].sum() == pytest.approx(expected)
def test_va_agentic_deflection_uses_residual():
"""Agentic VA must operate on the residual (1 − bot_rate) call pool,
not the full volume.
With bot_rate=0.35 and va_rate=0.15:
residual = 10,000 × (1 − 0.35) = 6,500 calls/mo
va_deflected = 6,500 × 0.15 = 975 calls/mo
"""
site = _small_site()
df = calculate_va_deflection_benefit(
[site],
FeatureScope("Agentic Virtual Agent", ["Small"], deflection_target=0.15),
"realistic",
year=1,
params="realistic",
)
completion = BENEFIT_PARAMS["va_completion_rate"]["realistic"]
labour = BENEFIT_PARAMS["va_labour_realization"]["realistic"]
callback = BENEFIT_PARAMS["va_callback_discount"]["realistic"]
real_factor = completion * labour * (1.0 - callback)
# realistic scenario: voice_bot_deflection = 0.35
bot_rate = 0.35
va_rate = 0.15
expected = (
10_000 * 12 # annual calls
* (1.0 - bot_rate) * va_rate # residual × va_rate (layered)
* 300 # AHT seconds
* 0.50 # Y1 scenario realization
* real_factor
* 0.01
)
assert df["annual_value"].sum() == pytest.approx(expected)
def test_va_no_double_count():
"""Combined bot + VA benefit must be less than the naive additive sum.
Naive (wrong): volume × (bot_rate + va_rate) × AHT × ...
Correct (layered): volume × (bot_rate + (1−bot_rate)×va_rate) × AHT × ...
With bot=35%, va=15%:
naive total deflection = 50%
layered total deflection = 35% + 65%×15% = 44.75%
"""
site = _small_site()
bot_scope = FeatureScope("Voice Bot", ["Small"], deflection_target=0.35)
va_scope = FeatureScope("Agentic Virtual Agent", ["Small"], deflection_target=0.15)
bot_df = calculate_va_deflection_benefit([site], bot_scope, "realistic", year=1)
va_df = calculate_va_deflection_benefit([site], va_scope, "realistic", year=1)
combined = bot_df["annual_value"].sum() + va_df["annual_value"].sum()
# Naive additive (the old broken model): both on full volume
completion = BENEFIT_PARAMS["va_completion_rate"]["realistic"]
labour = BENEFIT_PARAMS["va_labour_realization"]["realistic"]
callback = BENEFIT_PARAMS["va_callback_discount"]["realistic"]
real_factor = completion * labour * (1.0 - callback)
naive = (
10_000 * 12 * (0.35 + 0.15) * 300 * 0.50 * real_factor * 0.01
)
assert combined < naive, (
f"Combined layered benefit ({combined:.2f}) should be less than "
f"naive additive ({naive:.2f}) — double-count not fixed"
)
# Also verify the exact layered total
layered_deflection = 0.35 + (1.0 - 0.35) * 0.15 # = 0.4475
expected_combined = (
10_000 * 12 * layered_deflection * 300 * 0.50 * real_factor * 0.01
)
assert combined == pytest.approx(expected_combined)
def test_va_claim_params_reproduce_no_haircut():
"""params='claim' must apply zero haircuts (all factors = 1.0),
reproducing the original Genesys ROI-doc assumption."""
site = _small_site()
df_claim = calculate_va_deflection_benefit(
[site],
FeatureScope("Voice Bot", ["Small"], deflection_target=0.35),
"realistic",
year=1,
params="claim",
)
df_realistic = calculate_va_deflection_benefit(
[site],
FeatureScope("Voice Bot", ["Small"], deflection_target=0.35),
"realistic",
year=1,
params="realistic",
)
# claim should be strictly higher (no haircuts applied)
assert df_claim["annual_value"].sum() > df_realistic["annual_value"].sum()
# claim realization_factor = 1.0 × 1.0 × (1 − 0.0) = 1.0
expected_claim = 10_000 * 12 * 0.35 * 300 * 0.50 * 1.0 * 0.01
assert df_claim["annual_value"].sum() == pytest.approx(expected_claim)

2
studies/202512_GenesysCX/ctm-token-calculator/tests/test_business_case.py
View File

@@ -111,7 +111,7 @@ def test_scenario_json_roundtrip(tmp_path):
scenario_state_to_json(
CTM_DEFAULT_SITES, CTM_DEFAULT_TAKEOUTS, CTM_DEFAULT_FEATURE_SCOPES, p
)
sites, takeouts, scopes = scenario_state_from_json(p)
sites, takeouts, scopes, _rollout = scenario_state_from_json(p)
assert [s.site_name for s in sites] == [s.site_name for s in CTM_DEFAULT_SITES]
assert takeouts[0].annual_cost == CTM_DEFAULT_TAKEOUTS[0].annual_cost
assert scopes[0].adoption_curve == CTM_DEFAULT_FEATURE_SCOPES[0].adoption_curve

26
studies/202512_GenesysCX/ctm-token-calculator/tests/test_cost_model.py
View File

@@ -34,8 +34,8 @@ def test_default_sites_match_contracted_users():
def test_sta_acceptance_number():
"""2,088 users × 30 tokens × 12 months × $1 = $751,680."""
df = calculate_per_user_ai_cost(
CTM_DEFAULT_SITES, _scope("Speech & Text Analytics"),
DEFAULT_METERS["Speech & Text Analytics"], DEFAULT_PRICING,
CTM_DEFAULT_SITES, _scope("Speech & Text Analytics [named]"),
DEFAULT_METERS["Speech & Text Analytics [named]"], DEFAULT_PRICING,
)
assert df["annual_cost"].sum() == pytest.approx(751_680)
@@ -43,16 +43,20 @@ def test_sta_acceptance_number():
def test_agent_copilot_acceptance_number():
"""2,088 users × 40 tokens × 12 months × $1 = $1,002,240."""
df = calculate_per_user_ai_cost(
CTM_DEFAULT_SITES, _scope("Agent Copilot"),
DEFAULT_METERS["Agent Copilot"], DEFAULT_PRICING,
CTM_DEFAULT_SITES, _scope("Agent Copilot [named]"),
DEFAULT_METERS["Agent Copilot [named]"], DEFAULT_PRICING,
)
assert df["annual_cost"].sum() == pytest.approx(1_002_240)
def test_per_user_not_active_before_phase():
df = calculate_per_user_ai_cost(
CTM_DEFAULT_SITES, _scope("AI Translate", phase=3),
DEFAULT_METERS["AI Translate"], DEFAULT_PRICING, year=2,
def test_ai_translate_not_active_before_phase():
"""AI Translate (consumption meter) produces zero cost before its phase."""
scenario = get_scenario("realistic")
apac_sites = [s.site_name for s in CTM_DEFAULT_SITES if s.region_pricing == "APAC"]
df = calculate_consumption_ai_cost(
CTM_DEFAULT_SITES,
_scope("AI Translate", apac_sites, phase=3),
DEFAULT_METERS["AI Translate"], scenario, DEFAULT_PRICING, year=2,
)
assert df["annual_cost"].sum() == 0
@@ -63,7 +67,7 @@ def test_copilot_covers_supervisor_summary():
total = calculate_total_cost(
CTM_DEFAULT_SITES,
[
_scope("Agent Copilot"),
_scope("Agent Copilot [named]"),
_scope("AI Summary & Insights"),
],
DEFAULT_METERS, DEFAULT_PRICING, scenario, year=1,
@@ -111,8 +115,8 @@ def test_regional_pricing_not_hardcoded():
pricing["APAC"] = TokenPricing(region="APAC", list_rate_per_token=2.0)
apac_site = next(s for s in CTM_DEFAULT_SITES if s.region_pricing == "APAC")
df = calculate_per_user_ai_cost(
[apac_site], _scope("Speech & Text Analytics", [apac_site.site_name]),
DEFAULT_METERS["Speech & Text Analytics"], pricing,
[apac_site], _scope("Speech & Text Analytics [named]", [apac_site.site_name]),
DEFAULT_METERS["Speech & Text Analytics [named]"], pricing,
)
expected = apac_site.named_users * 30 * 12 * 2.0
assert df["annual_cost"].sum() == pytest.approx(expected)

42
studies/202512_GenesysCX/ctm-token-calculator/tests/test_meters.py
View File

@@ -10,10 +10,26 @@ from tokencalc.meters import Confidence, MeterType, TokenMeter, TokenPricing
def test_all_spec_meters_present():
expected = {
"Voice Bot", "Virtual Agent (legacy)", "Agentic Virtual Agent",
"AI Summary & Insights", "Direct Messaging", "Social Listening",
"Social Responses", "Speech & Text Analytics", "Agent Copilot",
"Email AI (Auto-Suggest)", "Email AI (Auto-Respond)", "AI Translate",
# Voice / Bot
"Voice Bot", "Digital Bot",
# Virtual Agent
"Virtual Agent (legacy)", "Agentic Virtual Agent",
# Agent Copilot (named + concurrent)
"Agent Copilot [named]", "Agent Copilot [concurrent]",
# AI Quality / Analytics
"AI Scoring", "AI Summary & Insights",
# Speech & Text Analytics (named + concurrent)
"Speech & Text Analytics [named]", "Speech & Text Analytics [concurrent]",
# Routing
"Predictive Routing",
# Messaging
"Direct Messaging", "Social Listening", "Social Responses",
# Language
"AI Translate",
# Genesys Cloud Copilot
"Genesys Cloud Copilot",
# Email AI (rates TBD)
"Email AI (Auto-Suggest)", "Email AI (Auto-Respond)",
}
assert expected == set(DEFAULT_METERS)
@@ -22,17 +38,29 @@ def test_confirmed_rates():
m = DEFAULT_METERS
assert m["Voice Bot"].units_per_token == 17
assert m["Voice Bot"].tokens_per_unit == pytest.approx(0.0588, abs=1e-3)
assert m["Digital Bot"].units_per_token == 51
assert m["Agentic Virtual Agent"].tokens_per_unit == 1.2
assert m["AI Summary & Insights"].tokens_per_unit == 0.02
assert m["Direct Messaging"].units_per_token == 400
assert m["Speech & Text Analytics"].tokens_per_unit == 30
assert m["Agent Copilot"].tokens_per_unit == 40
# Named variants
assert m["Speech & Text Analytics [named]"].tokens_per_unit == 30
assert m["Speech & Text Analytics [concurrent]"].tokens_per_unit == 45
assert m["Agent Copilot [named]"].tokens_per_unit == 40
assert m["Agent Copilot [concurrent]"].tokens_per_unit == 60
# AI Translate is now a confirmed consumption meter
assert m["AI Translate"].tokens_per_unit == 0.5
assert m["AI Translate"].units_per_token == 2
assert m["AI Translate"].confidence is Confidence.CONFIRMED
# New meters
assert m["AI Scoring"].units_per_token == 20
assert m["Predictive Routing"].units_per_token == 17
assert m["Genesys Cloud Copilot"].units_per_token == 20
def test_unknown_meters_flagged():
unknown = {f for f, m in DEFAULT_METERS.items() if m.confidence is Confidence.UNKNOWN}
assert unknown == {
"Email AI (Auto-Suggest)", "Email AI (Auto-Respond)", "AI Translate"
"Email AI (Auto-Suggest)", "Email AI (Auto-Respond)",
}
assert Confidence.UNKNOWN.icon == "🔴"
assert Confidence.CONFIRMED.icon == "🟢"