{ "generatedAt": "2026-10-26T17:59:24.492617+00:00", "window": { "start": "2026-10-12T16:42:47.757Z", "end": "2026-10-26T16:42:47.757Z" }, "languages": [ { "id": "zh-CN", "code": "zh-CN", "name": "Mandarin Chinese (Simplified Chinese)", "language": "Mandarin Chinese (Simplified Chinese)", "locale": "zh-CN", "native": "Mandarin Chinese (Simplified Chinese)" }, { "id": "ja-JP", "code": "ja-JP", "name": "Japanese", "language": "Japanese", "locale": "ja-JP", "native": "Japanese" }, { "id": "ko-KR", "code": "ko-KR", "name": "Korean", "language": "Korean", "locale": "ko-KR", "native": "Korean" }, { "id": "es-ES", "code": "es-ES", "name": "Spanish (Spain)", "language": "Spanish (Spain)", "locale": "es-ES", "native": "Spanish (Spain)" }, { "id": "fr-FR", "code": "fr-FR", "name": "French (France)", "language": "French (France)", "locale": "fr-FR", "native": "French (France)" }, { "id": "de-DE", "code": "de-DE", "name": "German (Germany)", "language": "German (Germany)", "locale": "de-DE", "native": "German (Germany)" }, { "id": "pt-BR", "code": "pt-BR", "name": "Brazilian Portuguese", "language": "Brazilian Portuguese", "locale": "pt-BR", "native": "Brazilian Portuguese" }, { "id": "ar", "code": "ar", "name": "Modern Standard Arabic", "language": "Modern Standard Arabic", "locale": "ar", "native": "Modern Standard Arabic" } ], "method": { "selection": "Top 20 per gender by distinct real registered callers in positive-duration private call transcripts over last 14 days; public non-retired characters with active Fish voice. Tie-break by call count then character id.", "generation": "40 previously selected public characters × 8 target-language passages × two requested models. Both arms use the same reference voice for a character/language pair; existing language-specific voices and the declared production fallback determine that reference. Matched text/settings; unmodified mono 24kHz PCM wrapped as WAV.", "judge": "Gemini 3.1 Pro hears opaque A/B audio with the original target text, language/locale and delivery direction; no names, models, portraits, gender, popularity or timings. Both presentation orders are reviewed. Per-language naturalness uses paired character means of two reviews and 5,000 character bootstrap resamples stratified by gender. The descriptive macro preserves complete character vectors across all languages.", "limitations": "One passage per language and one synthesis per model/reference/passage; one AI judge without native-listener validation. Accent variation is not automatically a defect. No voice-identity reference or live production-call latency measurement. Cross-language absolute scores are not directly comparable. Order reversals test reliability, not independent replication.", "notes": [ "Judge receives opaque pair ID, target language/locale, exact text/directions and audio only; no model names, identities, portraits, popularity, timings or file paths. Each pair gets two separate reversed-order judgments. Orchestrator holds labels; not independent human double-blind.", "Selection and English baseline are reused from the completed English study. Multilingual scripts, masking, control selection, transport settings and aggregation rules are frozen before new synthesis or judging.", "Existing locale-specific voice where present, else existing English bank slot, else active Fish slot; manually selected/cloned voices preserved. Each model pair always uses the exact same reference. This tests current character voices speaking the target language, not freshly selected native reference voices.", "Direct Fish WebSocket. One text message and flush for the full passage in every language, then stop; Fish may chunk internally. At most two synthesis sockets. No explicit language parameter: identical automatic language detection for both models. Fresh connection per sample.", "Confidence values average two uncalibrated model self-reports; they are not probabilities of a real-world win.", "Language-specific paired estimates are primary interpretation. The equal-language macro averages within each character and resamples whole character vectors, preserving dependence across languages. Absolute scores across languages are not directly comparable; the macro is descriptive only.", "Only one character has stored non-English language-specific reference slots; the other 39 use current cross-lingual character voices, not proven native-reference accents.", "All 40 selected characters remain visible; 0 sample-reviews are unassessable. Each per-language score requires both orders, and the macro requires every language per model/character; no null skipping or imputation.", "Calibration: 32/32 identical-audio reviews tied. Passing controls does not erase differing-audio order sensitivity.", "Judge retry accounting: 10 evaluations required retry, 13 failed API attempts and 685 minimum total calls. Failed API attempts do not establish billing charges.", "Masked A/B results were frozen before model decoding; the orchestrator knew the mapping, so this is not independent human double-blind.", "Direct cold WebSocket requests from this test host; no warm connection pool, not end-to-end production call latency.", "Synthesis failed attempts: 0; all 640 final WAVs decoded successfully.", "No production voice settings or existing rollout experiments were changed." ] }, "summary": { "characters": 40, "pairs": 320, "judgments": 640, "modelWins": { "s2-pro": 75, "s2.1-pro": 54 }, "ties": 50, "inconsistent": 141, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.7844, "s2.1-pro": 2.7703 }, "delta": -0.0141, "ci95": [ -0.1, 0.0734 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.5609, "s2.1-pro": 3.5656 }, "delta": 0.0047, "ci95": [ -0.0312, 0.0406 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.975, "s2.1-pro": 2.95 }, "delta": -0.025, "ci95": [ -0.0969, 0.0453 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.6953, "s2.1-pro": 3.6406 }, "delta": -0.0547, "ci95": [ -0.0969, -0.0187 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.0, "s2.1-pro": 3.9875 }, "delta": -0.0125, "ci95": [ -0.05, 0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 16, "reviews": 32, "ties": 32, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 32, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 16, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Equal-language macro: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 141/320 pairs changed preference across presentation order. Identical-audio controls tied 32/32 times; that does not establish order invariance on differing recordings.", "conclusion": "Equal-language macro: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 141/320 pairs changed preference across presentation order. Identical-audio controls tied 32/32 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 240, "B": 195, "tie": 205, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.0172, "oppositeWinners": 36, "tieVersusWinner": 105 }, "byLanguage": { "zh-CN": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 10, "s2.1-pro": 10 }, "ties": 5, "inconsistent": 15, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.975, "s2.1-pro": 3.0 }, "delta": 0.025, "ci95": [ -0.2125, 0.2625 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.6875, "s2.1-pro": 3.7125 }, "delta": 0.025, "ci95": [ -0.075, 0.125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.1125, "s2.1-pro": 3.0375 }, "delta": -0.075, "ci95": [ -0.2625, 0.1125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.775, "s2.1-pro": 3.7125 }, "delta": -0.0625, "ci95": [ -0.1375, 0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.0625, "s2.1-pro": 4.0625 }, "delta": 0.0, "ci95": [ -0.0375, 0.0375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Mandarin Chinese (Simplified Chinese): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 15/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Mandarin Chinese (Simplified Chinese): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 15/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 36, "B": 25, "tie": 19, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.15, "oppositeWinners": 6, "tieVersusWinner": 9 }, "languageId": "zh-CN", "language": "Mandarin Chinese (Simplified Chinese)", "locale": "zh-CN" }, "ja-JP": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 22, "s2.1-pro": 3 }, "ties": 2, "inconsistent": 13, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.175, "s2.1-pro": 2.8875 }, "delta": -0.2875, "ci95": [ -0.475, -0.1 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.8, "s2.1-pro": 3.65 }, "delta": -0.15, "ci95": [ -0.2375, -0.0625 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.3375, "s2.1-pro": 3.1375 }, "delta": -0.2, "ci95": [ -0.4125, 0.0125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.8625, "s2.1-pro": 3.7125 }, "delta": -0.15, "ci95": [ -0.2375, -0.075 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.075, "s2.1-pro": 3.7 }, "delta": -0.375, "ci95": [ -0.525, -0.225 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Japanese: Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 13/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Japanese: Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 13/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 38, "B": 28, "tie": 14, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.0625, "oppositeWinners": 3, "tieVersusWinner": 10 }, "languageId": "ja-JP", "language": "Japanese", "locale": "ja-JP" }, "ko-KR": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 11, "s2.1-pro": 12 }, "ties": 3, "inconsistent": 14, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.1, "s2.1-pro": 3.2625 }, "delta": 0.1625, "ci95": [ -0.075, 0.4 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.925, "s2.1-pro": 3.9125 }, "delta": -0.0125, "ci95": [ -0.0625, 0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.1625, "s2.1-pro": 3.2875 }, "delta": 0.125, "ci95": [ -0.075, 0.3375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.8125, "s2.1-pro": 3.775 }, "delta": -0.0375, "ci95": [ -0.1375, 0.05 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.1125, "s2.1-pro": 4.1125 }, "delta": 0.0, "ci95": [ 0.0, 0.0 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Korean: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 14/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Korean: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 14/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 34, "B": 34, "tie": 12, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": -0.0625, "oppositeWinners": 8, "tieVersusWinner": 6 }, "languageId": "ko-KR", "language": "Korean", "locale": "ko-KR" }, "es-ES": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 8, "s2.1-pro": 5 }, "ties": 8, "inconsistent": 19, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.425, "s2.1-pro": 2.5125 }, "delta": 0.0875, "ci95": [ -0.075, 0.2375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.325, "s2.1-pro": 3.3875 }, "delta": 0.0625, "ci95": [ -0.05, 0.175 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.75, "s2.1-pro": 2.7625 }, "delta": 0.0125, "ci95": [ -0.125, 0.1375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.6875, "s2.1-pro": 3.65 }, "delta": -0.0375, "ci95": [ -0.1375, 0.0625 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.1, "s2.1-pro": 4.125 }, "delta": 0.025, "ci95": [ 0.0, 0.0625 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Spanish (Spain): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 19/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Spanish (Spain): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 19/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 31, "B": 19, "tie": 30, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": -0.0125, "oppositeWinners": 5, "tieVersusWinner": 14 }, "languageId": "es-ES", "language": "Spanish (Spain)", "locale": "es-ES" }, "fr-FR": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 2, "s2.1-pro": 9 }, "ties": 9, "inconsistent": 20, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.35, "s2.1-pro": 2.5375 }, "delta": 0.1875, "ci95": [ 0.0375, 0.3375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.075, "s2.1-pro": 3.2 }, "delta": 0.125, "ci95": [ 0.05, 0.2125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.7375, "s2.1-pro": 2.9125 }, "delta": 0.175, "ci95": [ 0.075, 0.2875 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.5875, "s2.1-pro": 3.6375 }, "delta": 0.05, "ci95": [ -0.0125, 0.125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.95, "s2.1-pro": 3.9875 }, "delta": 0.0375, "ci95": [ 0.0, 0.0875 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "French (France): Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 20/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "French (France): Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 20/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 18, "B": 29, "tie": 33, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": -0.1625, "oppositeWinners": 5, "tieVersusWinner": 15 }, "languageId": "fr-FR", "language": "French (France)", "locale": "fr-FR" }, "de-DE": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 4, "s2.1-pro": 5 }, "ties": 7, "inconsistent": 24, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.725, "s2.1-pro": 2.7375 }, "delta": 0.0125, "ci95": [ -0.125, 0.15 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.5625, "s2.1-pro": 3.55 }, "delta": -0.0125, "ci95": [ -0.0625, 0.0375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.9, "s2.1-pro": 2.8875 }, "delta": -0.0125, "ci95": [ -0.1625, 0.125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.7125, "s2.1-pro": 3.675 }, "delta": -0.0375, "ci95": [ -0.125, 0.05 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.025, "s2.1-pro": 4.05 }, "delta": 0.025, "ci95": [ -0.05, 0.1125 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "German (Germany): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 24/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "German (Germany): No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 24/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 30, "B": 16, "tie": 34, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.1125, "oppositeWinners": 4, "tieVersusWinner": 20 }, "languageId": "de-DE", "language": "German (Germany)", "locale": "de-DE" }, "pt-BR": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 11, "s2.1-pro": 3 }, "ties": 7, "inconsistent": 19, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.8125, "s2.1-pro": 2.575 }, "delta": -0.2375, "ci95": [ -0.4, -0.0875 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.7, "s2.1-pro": 3.5875 }, "delta": -0.1125, "ci95": [ -0.2, -0.0375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.9, "s2.1-pro": 2.75 }, "delta": -0.15, "ci95": [ -0.275, -0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.7375, "s2.1-pro": 3.625 }, "delta": -0.1125, "ci95": [ -0.225, 0.0 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 4.025, "s2.1-pro": 4.1 }, "delta": 0.075, "ci95": [ 0.0125, 0.1625 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Brazilian Portuguese: Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 19/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Brazilian Portuguese: Naturalness estimates describe only these supplied recordings, not an established human-preference or production winner. 19/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 27, "B": 23, "tie": 30, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.0375, "oppositeWinners": 3, "tieVersusWinner": 16 }, "languageId": "pt-BR", "language": "Brazilian Portuguese", "locale": "pt-BR" }, "ar": { "characters": 40, "pairs": 40, "judgments": 80, "modelWins": { "s2-pro": 7, "s2.1-pro": 7 }, "ties": 9, "inconsistent": 17, "primary": { "metric": "Naturalness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.7125, "s2.1-pro": 2.65 }, "delta": -0.0625, "ci95": [ -0.1625, 0.0375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "secondary": { "intelligibility": { "metric": "intelligibility", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.4125, "s2.1-pro": 3.525 }, "delta": 0.1125, "ci95": [ -0.0125, 0.2375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "expressiveness": { "metric": "expressiveness", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 2.9, "s2.1-pro": 2.825 }, "delta": -0.075, "ci95": [ -0.175, 0.025 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "audioQuality": { "metric": "audioQuality", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.3875, "s2.1-pro": 3.3375 }, "delta": -0.05, "ci95": [ -0.15, 0.0375 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." }, "textFidelity": { "metric": "textFidelity", "scale": "1–5", "characters": 40, "excludedCharacters": 0, "genderCounts": { "Male": 20, "Female": 20 }, "modelMeans": { "s2-pro": 3.65, "s2.1-pro": 3.7625 }, "delta": 0.1125, "ci95": [ -0.0875, 0.325 ], "bootstrapResamples": 5000, "bootstrapSeed": 20261026, "intervalKind": "Descriptive paired character bootstrap, stratified by gender", "note": "All selected characters have every required score for this metric." } }, "calibration": { "pairs": 2, "reviews": 4, "ties": 4, "nonTies": 0, "unassessable": 0, "positionPreference": { "A": 0, "B": 0, "tie": 4, "unassessable": 0 }, "meanAbsoluteScoreGap": { "naturalness": 0.0, "intelligibility": 0.0, "expressiveness": 0.0, "audioQuality": 0.0, "textFidelity": 0.0 }, "maxAbsoluteScoreGap": { "naturalness": 0, "intelligibility": 0, "expressiveness": 0, "audioQuality": 0, "textFidelity": 0 }, "bothOrdersTiePairs": 2, "note": "Identical-audio controls should tie. Non-tie decisions and score gaps are disclosed as judge inconsistency, not filtered away." }, "note": "Modern Standard Arabic: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 17/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "conclusion": "Modern Standard Arabic: No clear S2.1 Pro naturalness advantage: the descriptive primary interval crosses zero. 17/40 pairs changed preference across presentation order. Identical-audio controls tied 4/4 times; that does not establish order invariance on differing recordings.", "unassessableReviews": 0, "orderBias": { "presentedPreference": { "A": 26, "B": 21, "tie": 33, "unassessable": 0 }, "meanPresentedANaturalnessMinusB": 0.0125, "oppositeWinners": 2, "tieVersusWinner": 15 }, "languageId": "ar", "language": "Modern Standard Arabic", "locale": "ar" } }, "voiceReferences": 42, "genderBalance": { "Male": 20, "Female": 20 }, "languageCount": 8, "samples": 640, "calibrationJudgments": 32, "totalJudgments": 672, "unassessableCases": [], "protocolNotes": "Selection and English baseline are reused from the completed English study. Multilingual scripts, masking, control selection, transport settings and aggregation rules are frozen before new synthesis or judging.", "maskedResultsSha256": "3d4cff7d7e7c59905a07d5d96c43995208e15e91e1b21f7e85d9f7dfa8e77487", "latency": { "method": "Direct cold WebSocket requests from this test host; no warm connection pool, not end-to-end production call latency.", "models": { "s2-pro": { "samples": 320, "medianTTFAms": 534.9, "p95TTFAms": 698.8, "meanTTFAms": 584.4, "medianAfterStartMs": 289.2, "p95AfterStartMs": 448.3, "medianOutputSeconds": 17.75, "failedAttempts": 0, "samplesRetried": 0 }, "s2.1-pro": { "samples": 320, "medianTTFAms": 514.8, "p95TTFAms": 1286.1, "meanTTFAms": 663.2, "medianAfterStartMs": 279.0, "p95AfterStartMs": 1053.1, "medianOutputSeconds": 17.66, "failedAttempts": 0, "samplesRetried": 0 } }, "failedAttempts": 0, "samplesRetried": 0, "note": "Timings describe successful attempts. Failed attempts are counted separately; requests were not concurrent within a matched pair." }, "modelVersions": { "gemini-3.1-pro-preview": 672 }, "successfulJudgmentTokens": { "promptTokenCount": 1363094, "candidatesTokenCount": 313616, "thoughtsTokenCount": 375079, "totalTokenCount": 2051789 }, "judgeRetryEvaluations": 10, "judgeFailedAPIAttempts": 13, "judgeMinimumAPICalls": 685, "judgeRetryNote": "Failed provider API attempts were retried with unchanged inputs, including one request recovered after three HTTP500 failures. Failed attempts do not establish a billed charge; no quality-based rejudging occurred.", "confidenceNote": "Confidence values average two uncalibrated model self-reports; they are not probabilities of a real-world win.", "macroInterpretation": "Language-specific paired estimates are primary interpretation. The equal-language macro averages within each character and resamples whole character vectors, preserving dependence across languages. Absolute scores across languages are not directly comparable; the macro is descriptive only." }, "characters": [ { "id": "m01", "name": "Jordan", "slug": "jordan-1a5496", "gender": "Male", "rank": 1, "portrait": "portraits/m01.webp", "language": "8 languages", "popularity": { "callers": 29, "calls": 43 }, "pairs": [ { "id": "m01-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-zh-CN-A.wav", "model": "s2.1-pro", "duration": 17.2727 }, "b": { "src": "audio/m01-zh-CN-B.wav", "model": "s2-pro", "duration": 17.203 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are highly intelligible and accurate to the text but lack the requested warmth and teasing tone. They sound similarly stiff and robotic, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text but lack the requested warmth and teasing tone. They sound similarly stiff and robotic, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. However, both struggle to convey the requested warmth and teasing tone, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The tone is consistent throughout, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": 13.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and tone are slightly mechanical, missing the intended warmth and gentle teasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery remains relatively flat and does not clearly shift to a teasing tone.", "severity": "minor", "timestamp_seconds": 12.0 } ] }, "limitations": [ "Both samples exhibit a somewhat robotic and flat delivery, making it difficult to distinguish a clear winner in expressiveness.", "Both samples exhibit a somewhat flat delivery, not fully capturing the requested emotional shifts." ] } }, { "id": "m01-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-ja-JP-A.wav", "model": "s2-pro", "duration": 18.6891 }, "b": { "src": "audio/m01-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.0606 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A pronounces the text correctly, while Sample B mispronounces the first word. Both samples suffer from unnatural pitch accents, but A is more accurate.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A pronounces the text correctly, while Sample B mispronounces the first word. Both samples suffer from unnatural pitch accents, but A is more accurate." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides a more natural delivery with correct pronunciation and captures the teasing tone well. Sample A mispronounces the first word." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Heavy foreign accent and incorrect pitch accents throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'isoganakute' as 'isaganakute'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Heavy foreign accent and incorrect pitch accents throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'isaganakute'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing and intonation feel somewhat stiff and unnatural.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is assessing Japanese pitch accent and pronunciation." ] } }, { "id": "m01-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-ko-KR-A.wav", "model": "s2.1-pro", "duration": 17.4352 }, "b": { "src": "audio/m01-ko-KR-B.wav", "model": "s2-pro", "duration": 18.7355 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A delivers a much more natural and expressive performance, capturing the requested warm and teasing tones. Sample B is highly robotic and flat, failing to meet the delivery directions.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A delivers a much more natural and expressive performance, capturing the requested warm and teasing tones. Sample B is highly robotic and flat, failing to meet the delivery directions." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly better naturalness and expressiveness, capturing the requested tone shift more effectively than Sample A." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Minor synthetic cadence in some transitions.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are highly unnatural and stilted throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is flat and lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff phrasing and unnatural pauses.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m01-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-es-ES-A.wav", "model": "s2-pro", "duration": 16.2742 }, "b": { "src": "audio/m01-es-ES-B.wav", "model": "s2.1-pro", "duration": 16.1814 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to produce a native Spanish (Spain) accent, instead reading the text with a heavy English accent. Neither captures the requested emotional delivery.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples fail to produce a native Spanish (Spain) accent, instead reading the text with a heavy English accent. Neither captures the requested emotional delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples feature a noticeable non-native accent, but Sample A offers more fluid and natural prosody. Sample B is noticeably choppy and robotic." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, such as pronouncing 'He' with an English 'h' sound and using English rhotic 'r's.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or gently teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural cadence.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent applied to Spanish text, including incorrect vowel sounds and English consonants.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is flat and robotic, missing the warm and teasing directions.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a severe English accent, making it difficult to assess native Spanish prosody.", "Both samples exhibit a strong non-native accent for the requested locale." ] } }, { "id": "m01-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-fr-FR-A.wav", "model": "s2.1-pro", "duration": 14.8578 }, "b": { "src": "audio/m01-fr-FR-B.wav", "model": "s2-pro", "duration": 15.6937 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples feature a strong non-native accent, but Sample A has slightly clearer audio quality and more natural phrasing. Sample B sounds slightly more muffled.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples feature a strong non-native accent, but Sample A has slightly clearer audio quality and more natural phrasing. Sample B sounds slightly more muffled." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides a much more natural and accurate French pronunciation. Sample A suffers from a very strong English accent that degrades naturalness and intelligibility." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, affecting vowels and nasal sounds.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, affecting words like 'fenêtre' and 'tranquille'.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'réservé' and 'fenêtre'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a heavy non-native accent, limiting the evaluation of native-level French nuances." ] } }, { "id": "m01-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-de-DE-A.wav", "model": "s2.1-pro", "duration": 18.1086 }, "b": { "src": "audio/m01-de-DE-B.wav", "model": "s2-pro", "duration": 17.6674 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a strong non-native accent. Sample A is preferred because its phrasing and prosody flow more naturally, whereas Sample B sounds more disjointed and robotic.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples exhibit a strong non-native accent. Sample A is preferred because its phrasing and prosody flow more naturally, whereas Sample B sounds more disjointed and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a noticeable foreign accent. Sample A is preferred for slightly better prosody and expressiveness, whereas Sample B sounds more robotic." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and 'r' sounds.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent throughout, affecting vowels and 'r' sounds.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth and teasing tone.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and 'r' sounds.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is choppy and disjointed, lacking natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign accent on words like 'reserviert' and 'Fenster'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Cadence is somewhat stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, which affects naturalness." ] } }, { "id": "m01-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-pt-BR-A.wav", "model": "s2-pro", "duration": 19.0142 }, "b": { "src": "audio/m01-pt-BR-B.wav", "model": "s2.1-pro", "duration": 19.0838 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a heavy foreign accent when speaking Portuguese, making them sound unnatural for the requested locale. They are tied in overall quality and fidelity.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy foreign accent when speaking Portuguese, making them sound unnatural for the requested locale. They are tied in overall quality and fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a noticeable foreign accent, but Sample A is more natural and expressive. Sample B is choppy and lacks the requested warmth." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent, e.g., 'amanhã' sounds distorted.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural cadence.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout the sample.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent that affects pronunciation.", "Both samples exhibit a strong non-native accent for Brazilian Portuguese." ] } }, { "id": "m01-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m01-ar-A.wav", "model": "s2-pro", "duration": 18.9213 }, "b": { "src": "audio/m01-ar-B.wav", "model": "s2.1-pro", "duration": 19.7108 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit similar prosodic limitations and read the time in a colloquial manner. Neither clearly outperforms the other in naturalness or expressiveness.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar prosodic limitations and read the time in a colloquial manner. Neither clearly outperforms the other in naturalness or expressiveness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Both samples suffer from an unnatural pause in the middle of a phrase. Sample A is preferred for a slightly more natural vocal tone and better expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafitha'.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Reads 7:45 as 'sab'a khamsa wa arba'in' rather than formal MSA time.", "severity": "minor", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafitha'.", "severity": "minor", "timestamp_seconds": 7.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Reads 7:45 as 'sab'a khamsa wa arba'in' rather than formal MSA time.", "severity": "minor", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 } ] }, "limitations": [ "Evaluation of Arabic time formatting is based on common conversational norms rather than strict MSA grammar.", "Non-native Arabic speaker; evaluation relies on general phonetic and prosodic principles." ] } } ] }, { "id": "m02", "name": "Alex Park", "slug": "alex-park-404cbc", "gender": "Male", "rank": 2, "portrait": "portraits/m02.webp", "language": "8 languages", "popularity": { "callers": 24, "calls": 30 }, "pairs": [ { "id": "m02-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.7851 }, "b": { "src": "audio/m02-zh-CN-B.wav", "model": "s2-pro", "duration": 17.6907 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a very strong non-native accent with frequent tonal errors. They are equally intelligible but lack naturalness.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a very strong non-native accent with frequent tonal errors. They are equally intelligible but lack naturalness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent with frequent tone errors. Sample B is preferred because it has slightly better fluidity and phrasing, whereas Sample A sounds more choppy." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent and incorrect tones throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with incorrect tones throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent and incorrect tones throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with incorrect tones throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy delivery with unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, making subtle expressiveness difficult to evaluate.", "Both samples feature a strong non-native accent, making tone evaluation challenging." ] } }, { "id": "m02-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-ja-JP-A.wav", "model": "s2-pro", "duration": 18.2479 }, "b": { "src": "audio/m02-ja-JP-B.wav", "model": "s2.1-pro", "duration": 18.991 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from the same major pronunciation error at the beginning and exhibit a strong non-native accent with robotic prosody. Neither is clearly better.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from the same major pronunciation error at the beginning and exhibit a strong non-native accent with robotic prosody. Neither is clearly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from severe pronunciation errors that change word meanings. Sample A is slightly more expressive and less robotic than B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "2.5/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'ikanakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Strong foreign accent and unnatural cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '急がなくて' (isoganakute) as '行かなくて' (ikanakute).", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '明日の' (ashita no) as 'あすたの' (asuta no).", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'ikanakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Strong foreign accent and unnatural cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '急がなくて' (isoganakute) as '行かなくて' (ikanakute).", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '明日の' (ashita no) as 'あさの' (asa no).", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, which affects naturalness.", "Evaluator is not a native Japanese speaker, but pronunciation errors are clear." ] } }, { "id": "m02-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-ko-KR-A.wav", "model": "s2-pro", "duration": 19.8037 }, "b": { "src": "audio/m02-ko-KR-B.wav", "model": "s2.1-pro", "duration": 18.8981 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better phrasing and flow. Sample A is noticeably more choppy and robotic in its delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better phrasing and flow. Sample A is noticeably more choppy and robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear pronunciation and good audio quality. However, both are somewhat flat and do not strongly convey the requested emotional shifts, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy phrasing and unnatural pauses throughout the utterance.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The emotional shift to teasing is subtle and lacks dynamic range.", "severity": "minor", "timestamp_seconds": 14.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural intonation and slightly robotic cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to 'gently teasing' is not very distinct.", "severity": "minor", "timestamp_seconds": 13.0 } ] }, "limitations": [ "Both samples exhibit a strong non-native accent or robotic cadence, making naturalness evaluation challenging.", "I am not a native Korean speaker, but I can assess general pronunciation, text fidelity, and audio quality." ] } }, { "id": "m02-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-es-ES-A.wav", "model": "s2-pro", "duration": 17.5049 }, "b": { "src": "audio/m02-es-ES-B.wav", "model": "s2.1-pro", "duration": 17.0405 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a heavy English accent when speaking Spanish, with similar levels of intelligibility and expressiveness. Neither is clearly superior.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy English accent when speaking Spanish, with similar levels of intelligibility and expressiveness. Neither is clearly superior." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a very heavy foreign accent that significantly impacts naturalness and intelligibility. Neither successfully captures a native Spanish pronunciation, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., pronouncing the silent 'h' in 'He reservado'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'hablar' and 'tranquilo'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., pronouncing the silent 'h' in 'He reservado'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'tarde' and 'galleta'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native (American English) accent, making it difficult to assess native Spanish prosody.", "Both samples exhibit a very strong English accent when reading Spanish, making it difficult to assess native-level prosody." ] } }, { "id": "m02-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-fr-FR-A.wav", "model": "s2-pro", "duration": 15.5776 }, "b": { "src": "audio/m02-fr-FR-B.wav", "model": "s2.1-pro", "duration": 15.7866 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a very strong English accent that severely impacts the naturalness and intelligibility of the French text. Neither successfully captures the requested delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a very strong English accent that severely impacts the naturalness and intelligibility of the French text. Neither successfully captures the requested delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use French phonology, reading the text with a heavy English accent. Neither is usable for a native French scenario." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Prends ton temps' sounds like 'Prunes tone tawn'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent applied to French words (e.g., 'pruns tone tom' for 'prends ton temps').", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Prends ton temps' sounds like 'Pruns tone tawn'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent applied to French words (e.g., 'pruns tone tom' for 'prends ton temps').", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a heavy non-native accent.", "Both samples exhibit a severe foreign (English) accent, making it difficult to assess native French prosody." ] } }, { "id": "m02-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-de-DE-A.wav", "model": "s2-pro", "duration": 17.621 }, "b": { "src": "audio/m02-de-DE-B.wav", "model": "s2.1-pro", "duration": 18.8284 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A has slightly better pacing and flow compared to Sample B, which features somewhat stilted pauses. Both lack a strong shift to gentle teasing.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has slightly better pacing and flow compared to Sample B, which features somewhat stilted pauses. Both lack a strong shift to gentle teasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a very strong English accent that severely impacts the naturalness and pronunciation of the German text. Neither is clearly superior." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Slight mispronunciation of 'reden' sounding closer to 'rieden'.", "severity": "minor", "timestamp_seconds": 9.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks a clear shift to gentle teasing at the end.", "severity": "minor", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent affects multiple words, such as 'reden' and 'genießen'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and slightly stilted pacing throughout.", "severity": "minor", "timestamp_seconds": 3.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not convey the requested gentle teasing.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent affects multiple words, such as 'reden' and 'genießen'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional accents in German is limited." ] } }, { "id": "m02-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-pt-BR-A.wav", "model": "s2-pro", "duration": 18.4569 }, "b": { "src": "audio/m02-pt-BR-B.wav", "model": "s2.1-pro", "duration": 19.1767 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples successfully deliver the target text with clear audio quality but suffer from a noticeable non-native accent and somewhat robotic prosody. Neither effectively captures the requested emotional shifts.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text with clear audio quality but suffer from a noticeable non-native accent and somewhat robotic prosody. Neither effectively captures the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong foreign accent that severely impacts the naturalness and pronunciation of the Brazilian Portuguese text. Neither sounds like a native speaker, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, e.g., 'amanhã' and 'tranquilo' are mispronounced.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects the naturalness of the Portuguese.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm, reassuring, and teasing emotions requested.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, e.g., 'amanhã' and 'tranquilo' are mispronounced.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, affecting naturalness.", "Both samples exhibit a heavy foreign accent, making it difficult to assess native-level prosody." ] } }, { "id": "m02-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m02-ar-A.wav", "model": "s2-pro", "duration": 20.2681 }, "b": { "src": "audio/m02-ar-B.wav", "model": "s2.1-pro", "duration": 20.3145 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples feature a heavy non-native accent with several pronunciation errors. Sample B is preferred because it correctly translates and pronounces the time in Arabic, whereas Sample A uses English numbers.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples feature a heavy non-native accent with several pronunciation errors. Sample B is preferred because it correctly translates and pronounces the time in Arabic, whereas Sample A uses English numbers." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A maintains the target language throughout, despite some pronunciation inaccuracies. Sample B reads part of the time in English, which is a major text fidelity failure." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "2.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "wrong_language", "evidence": "Reads '7:45' using English numbers instead of Arabic.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'معًا' and 'قطعة'.", "severity": "major", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "wrong_language", "evidence": "Reads '45' as English 'forty five'.", "severity": "blocking", "timestamp_seconds": 4.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'معًا' and 'قطعة'.", "severity": "major", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'النافذة' and 'هادئة'.", "severity": "major", "timestamp_seconds": 8.0 } ] }, "limitations": [ "Evaluator is not a native speaker of Modern Standard Arabic.", "I am not a native speaker of Arabic, but the English number insertion in B is clear." ] } } ] }, { "id": "m03", "name": "Declan Ward", "slug": "declan-ward-1fe89c", "gender": "Male", "rank": 3, "portrait": "portraits/m03.webp", "language": "8 languages", "popularity": { "callers": 20, "calls": 26 }, "pairs": [ { "id": "m03-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-zh-CN-A.wav", "model": "s2-pro", "duration": 17.6674 }, "b": { "src": "audio/m03-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.7154 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural and warm delivery that better fits the requested tone. Sample B feels slightly rushed and stiff in comparison.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and warm delivery that better fits the requested tone. Sample B feels slightly rushed and stiff in comparison." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers more natural pacing and better reflects the intended warm tone. Sample B is overly slow and monotonous, failing to capture the requested expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is extremely slow and deliberate, lacking natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones, sounding monotonous instead.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing feels slightly rushed and less natural.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and pauses feel slightly disjointed and unnatural.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m03-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-ja-JP-A.wav", "model": "s2-pro", "duration": 22.2185 }, "b": { "src": "audio/m03-ja-JP-B.wav", "model": "s2.1-pro", "duration": 20.1752 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A captures the requested warm and reassuring tone effectively. Sample B sounds overly distressed and breathy, missing the intended delivery.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A captures the requested warm and reassuring tone effectively. Sample B sounds overly distressed and breathy, missing the intended delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural and appropriate delivery for the requested warm and reassuring tone. Sample A sounds overly dramatic and breathless, detracting from the intended mood." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker sounds distressed and out of breath rather than warm and reassuring.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery sounds overly dramatic and breathless rather than warm and reassuring.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m03-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-ko-KR-A.wav", "model": "s2.1-pro", "duration": 18.4105 }, "b": { "src": "audio/m03-ko-KR-B.wav", "model": "s2-pro", "duration": 18.8981 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred for its more consistent delivery. Sample B uses an overly breathy, almost whispering tone that does not fit the 'warm, reassuring' prompt, and has an abrupt volume shift.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred for its more consistent delivery. Sample B uses an overly breathy, almost whispering tone that does not fit the 'warm, reassuring' prompt, and has an abrupt volume shift." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a warmer and more natural delivery that better matches the requested tone. Sample B suffers from slightly choppy phrasing and less convincing expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses between sentences.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly choppy phrasing and unnatural pauses between clauses.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and whispering tone does not sound warm.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "AB", "category": "other", "evidence": "Abrupt shift in volume and tone.", "severity": "minor", "timestamp_seconds": 15.0 } ] }, "limitations": [ "I am not a native Korean speaker, but I can evaluate general prosody and text fidelity." ] } }, { "id": "m03-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-es-ES-A.wav", "model": "s2-pro", "duration": 16.5529 }, "b": { "src": "audio/m03-es-ES-B.wav", "model": "s2.1-pro", "duration": 17.2727 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural and appropriate delivery for the requested warm and reassuring tone. Sample B sounds overly breathy and slightly strained, detracting from the intended mood.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and appropriate delivery for the requested warm and reassuring tone. Sample B sounds overly breathy and slightly strained, detracting from the intended mood." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and pacing. Sample A sounds more robotic and breathy, though both struggle to convey the requested warmth and teasing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly rigid pacing in the first half.", "severity": "minor", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and does not fully capture the requested warmth or teasing.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice sounds strained and overly breathy, missing the warm and reassuring tone.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The voice sounds breathy and unnatural throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or gently teasing tones.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a somewhat robotic tone, making it difficult to fully assess the intended emotional delivery." ] } }, { "id": "m03-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-fr-FR-A.wav", "model": "s2.1-pro", "duration": 16.669 }, "b": { "src": "audio/m03-fr-FR-B.wav", "model": "s2-pro", "duration": 18.3176 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody and strong non-native accents that detract from the naturalness of the French text. Neither successfully captures the requested warm or teasing delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and strong non-native accents that detract from the naturalness of the French text. Neither successfully captures the requested warm or teasing delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from highly unnatural, breathy delivery that sounds eerie rather than warm and reassuring. Sample B is slightly preferred as its pacing is marginally less disjointed than Sample A." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is highly disjointed and unnatural throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects words like 'Prends'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and lacks the requested warm, reassuring tone.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Intonation and pacing remain unnatural, though slightly better connected than A.", "severity": "major", "timestamp_seconds": 3.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is unnaturally slow and disjointed.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is excessively breathy and slow, sounding eerie rather than warm and reassuring.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is highly disjointed with unnatural pauses between words.", "severity": "major", "timestamp_seconds": 2.0 } ] }, "limitations": [ "Both samples feature strong non-native accents and robotic prosody, making it difficult to distinguish a clear winner." ] } }, { "id": "m03-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-de-DE-A.wav", "model": "s2-pro", "duration": 21.2665 }, "b": { "src": "audio/m03-de-DE-B.wav", "model": "s2.1-pro", "duration": 19.316 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit highly robotic, disjointed prosody and fail to capture the requested warm and teasing delivery. They are equally unnatural.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples exhibit highly robotic, disjointed prosody and fail to capture the requested warm and teasing delivery. They are equally unnatural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples fail to normalize the time correctly and have strong non-native accents. Sample A is preferred for having slightly better fluency and fewer awkward pauses than Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Highly disjointed and unnatural phrasing throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_substitution", "evidence": "Reads '19:45' as '19 Uhr 5 und 40' instead of '19 Uhr 45'.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pauses and breathiness disrupt the flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects intelligibility slightly.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy and unnatural cadence throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not sound warm or gently teasing as requested.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_substitution", "evidence": "Reads '19:45' as '19 Uhr 5 und 40' instead of '19 Uhr 45'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both models exhibit a strong non-native (American) accent in German." ] } }, { "id": "m03-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-pt-BR-A.wav", "model": "s2.1-pro", "duration": 20.7789 }, "b": { "src": "audio/m03-pt-BR-B.wav", "model": "s2-pro", "duration": 18.5266 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples use a European Portuguese accent rather than the requested Brazilian Portuguese. Sample A is preferred because its pacing is slightly more natural, whereas Sample B is overly slow and disjointed.", "confidence": 0.775, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples use a European Portuguese accent rather than the requested Brazilian Portuguese. Sample A is preferred because its pacing is slightly more natural, whereas Sample B is overly slow and disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.85, "rationale": "Sample A provides a more natural and expressive delivery that aligns well with the requested warm tone. Sample B suffers from slightly robotic and disjointed phrasing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff with unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 19.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is somewhat disjointed and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is very slow and disjointed, lacking natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 17.0 } ] }, "limitations": [] } }, { "id": "m03-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m03-ar-A.wav", "model": "s2.1-pro", "duration": 20.3145 }, "b": { "src": "audio/m03-ar-B.wav", "model": "s2-pro", "duration": 20.9183 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred due to better text fidelity. Sample B mispronounces the time, omitting 'five' from '7:45', and alters the verb conjugation. Both suffer from robotic cadence.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to better text fidelity. Sample B mispronounces the time, omitting 'five' from '7:45', and alters the verb conjugation. Both suffer from robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from unnatural pauses and robotic prosody. Sample A mispronounces 'حجزت' as 'hajazat' (she booked), while Sample B mispronounces 'مقعدين' and omits a conjunction in the time." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural cadence and phrasing throughout, with an awkward pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'مقعدين' as 'mag'adin'.", "severity": "major", "timestamp_seconds": 2.8 }, { "order": "BA", "category": "text_omission", "evidence": "Omits 'wa' in the time, saying 'sab'a khamsa wa arba'in'.", "severity": "minor", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.0 } ], "B": [ { "order": "AB", "category": "text_omission", "evidence": "Omits 'five' when reading '7:45', saying 'سبعة وأربعين' instead of 'سبعة وخمسة وأربعين'.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural cadence and an awkward pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 } ] }, "limitations": [ "Both samples exhibit significant robotic prosody and unnatural pauses, making them difficult to differentiate on naturalness.", "Both samples exhibit unnatural pauses and robotic prosody." ] } } ] }, { "id": "m04", "name": "A femboy waiting for you to call him goodboy", "slug": "a-femboy-waiting-for-you-to-call-him-goodboy-e8c958", "gender": "Male", "rank": 4, "portrait": "portraits/m04.webp", "language": "8 languages", "popularity": { "callers": 18, "calls": 27 }, "pairs": [ { "id": "m04-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-zh-CN-A.wav", "model": "s2.1-pro", "duration": 18.2247 }, "b": { "src": "audio/m04-zh-CN-B.wav", "model": "s2-pro", "duration": 19.1071 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from severe robotic prosody, flat tones, and a strong non-native accent. Neither successfully conveys the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody, flat tones, and a strong non-native accent. Neither successfully conveys the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit highly robotic and disjointed prosody, failing to capture the requested warm and teasing delivery. They are equally intelligible but lack naturalness." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech has a very flat, unnatural cadence with poor tone realization throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or teasing emotions due to flat delivery.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is highly unnatural, with disjointed phrasing and flat pitch.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is completely flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech is characterized by flat tones and an unnatural, choppy rhythm.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The flat, robotic delivery completely misses the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is disjointed with unnatural pauses and flat intonation throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, and gently teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent and flat tones, making it difficult to assess nuanced expressiveness." ] } }, { "id": "m04-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-ja-JP-A.wav", "model": "s2.1-pro", "duration": 21.9863 }, "b": { "src": "audio/m04-ja-JP-B.wav", "model": "s2-pro", "duration": 19.8037 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is slightly preferred as it avoids a minor pronunciation error found in Sample A, though both suffer from robotic prosody.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is slightly preferred as it avoids a minor pronunciation error found in Sample A, though both suffer from robotic prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with high fidelity and clear intelligibility. However, both suffer from a similarly robotic and unnatural prosody, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Slight mispronunciation of 'クッキー' (cookie), sounding like 'kutkii'.", "severity": "minor", "timestamp_seconds": 19.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and robotic cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech pattern is stiff and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and robotic cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall delivery is somewhat monotone and lacks natural human-like cadence.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit robotic prosody and unnatural pausing.", "Both samples exhibit a somewhat robotic and unnatural prosody, making it difficult to distinguish a clear winner." ] } }, { "id": "m04-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-ko-KR-A.wav", "model": "s2.1-pro", "duration": 20.1288 }, "b": { "src": "audio/m04-ko-KR-B.wav", "model": "s2-pro", "duration": 20.152 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural phrasing and better captures the warm tone requested. Sample A is a bit more robotic in its cadence.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly more natural phrasing and better captures the warm tone requested. Sample A is a bit more robotic in its cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly more natural pacing and intonation. Sample B sounds a bit more robotic and disjointed in its phrasing. Both have similar audio quality and perfect text fidelity." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff pacing throughout the utterance.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing feels slightly disjointed and unnatural.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat delivery, lacking strong emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, but I can assess general pronunciation, pacing, and audio quality." ] } }, { "id": "m04-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-es-ES-A.wav", "model": "s2.1-pro", "duration": 19.0142 }, "b": { "src": "audio/m04-es-ES-B.wav", "model": "s2-pro", "duration": 17.7603 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a severe English accent when speaking Spanish, failing to meet the locale expectations. They are equally unnatural and robotic.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a severe English accent when speaking Spanish, failing to meet the locale expectations. They are equally unnatural and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong non-native accent and robotic, disjointed prosody. Neither successfully captures the requested delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'reservado' and 'galleta'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth and teasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is stiff and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent, with English-like 'r' sounds in 'reservado'.", "severity": "major", "timestamp_seconds": 2.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and unnatural, missing the emotional cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is disjointed with unnatural pauses between phrases.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'reservado' and 'tranquilo'.", "severity": "major", "timestamp_seconds": 2.5 } ] }, "limitations": [ "Evaluator is not a native speaker of Spain Spanish, but the English accent is universally obvious.", "Both samples exhibit a strong non-native accent, making it difficult to assess native-level Spanish prosody." ] } }, { "id": "m04-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-fr-FR-A.wav", "model": "s2.1-pro", "duration": 16.3671 }, "b": { "src": "audio/m04-fr-FR-B.wav", "model": "s2-pro", "duration": 16.3671 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a very strong non-native accent that distorts pronunciation. Sample A is preferred for having slightly more natural phrasing and expressiveness compared to B's flatter delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a very strong non-native accent that distorts pronunciation. Sample A is preferred for having slightly more natural phrasing and expressiveness compared to B's flatter delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe non-native accent that heavily impacts naturalness and pronunciation. Neither successfully captures the requested warm or teasing delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, distorting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'réservé' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, distorting words like 'fenêtre' and 'tranquille'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is somewhat flat and mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluated by a non-native speaker, though the heavy English accent is unmistakable.", "Both samples exhibit a very strong non-native accent, making it difficult to assess subtle prosodic features." ] } }, { "id": "m04-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-de-DE-A.wav", "model": "s2.1-pro", "duration": 19.2928 }, "b": { "src": "audio/m04-de-DE-B.wav", "model": "s2-pro", "duration": 19.8733 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from robotic prosody and a strong non-native accent. Sample B is slightly preferred for better initial clarity.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from robotic prosody and a strong non-native accent. Sample B is slightly preferred for better initial clarity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong foreign accent and mispronounce 'Keks' as the English word 'cakes'. Neither successfully conveys the requested warm or teasing emotions, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 17.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 17.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speech is robotic and does not reflect the requested emotional shifts.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural phrasing and cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 17.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 18.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent that affects pronunciation.", "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "m04-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-pt-BR-A.wav", "model": "s2-pro", "duration": 19.525 }, "b": { "src": "audio/m04-pt-BR-B.wav", "model": "s2.1-pro", "duration": 20.9647 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples perform similarly, with comparable audio quality, intelligibility, and text fidelity. Neither clearly outperforms the other in naturalness or expressiveness.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples perform similarly, with comparable audio quality, intelligibility, and text fidelity. Neither clearly outperforms the other in naturalness or expressiveness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with similar intelligibility and audio quality. However, both struggle with the requested Brazilian Portuguese locale, sounding more like European Portuguese or non-native speakers. Neither successfully captures the requested emotional shifts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly unnatural and mechanical throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warmth and teasing tone, sounding somewhat robotic.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Exhibits a strong non-native or European Portuguese accent instead of Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery lacks natural human cadence, sounding somewhat mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is relatively flat and does not clearly shift from warm to gently teasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The accent leans heavily towards European Portuguese rather than the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent or robotic cadence.", "Both samples exhibit a non-native or European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "m04-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m04-ar-A.wav", "model": "s2.1-pro", "duration": 22.4972 }, "b": { "src": "audio/m04-ar-B.wav", "model": "s2-pro", "duration": 23.2635 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from severe robotic prosody, synthetic artifacts, and unnatural pauses (e.g., before 'an-naafidha'). Neither successfully conveys the requested emotional shifts, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody, synthetic artifacts, and unnatural pauses (e.g., before 'an-naafidha'). Neither successfully conveys the requested emotional shifts, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from highly robotic, disjointed prosody with unnatural pauses mid-sentence. Neither successfully conveys the requested warm or teasing delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 2.5 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The overall delivery is highly robotic with unnatural cadence and flat intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested 'warm, reassuring' or 'gently teasing' tones.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "Unnatural pause between 'qurb' and 'an-naafidha'.", "severity": "minor", "timestamp_seconds": 9.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed phrasing with awkward pauses, such as before 'النافذة'.", "severity": "major", "timestamp_seconds": 9.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech is very robotic, lacking natural human cadence and phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is flat and does not reflect the intended warmth or teasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "Awkward pause and phrasing around 'qurbal... an-naafidha'.", "severity": "minor", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly disjointed delivery with unnatural pauses, such as between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'لنترك' as 'lanatruk' instead of 'linatruk'.", "severity": "minor", "timestamp_seconds": 18.0 } ] }, "limitations": [ "Both samples exhibit significant robotic artifacts and unnatural prosody, making it difficult to assess fine-grained expressiveness.", "Both samples exhibit severe robotic prosody and unnatural pauses, making them difficult to differentiate in quality." ] } } ] }, { "id": "m05", "name": "Damian Crowe", "slug": "damian-crowe-2647a6", "gender": "Male", "rank": 5, "portrait": "portraits/m05.webp", "language": "8 languages", "popularity": { "callers": 17, "calls": 26 }, "pairs": [ { "id": "m05-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-zh-CN-A.wav", "model": "s2-pro", "duration": 16.9476 }, "b": { "src": "audio/m05-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.251 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately but suffer from flat, robotic prosody. Neither successfully captures the requested warm or teasing emotional shifts.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately but suffer from flat, robotic prosody. Neither successfully captures the requested warm or teasing emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better naturalness and expressiveness, capturing the intended warm tone more effectively than Sample B, which sounds somewhat monotonous." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native speaker of Mandarin Chinese, but can assess general prosody and text fidelity." ] } }, { "id": "m05-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.1303 }, "b": { "src": "audio/m05-ja-JP-B.wav", "model": "s2-pro", "duration": 20.5003 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B correctly pronounces 'ashita' (tomorrow), whereas Sample A incorrectly reads the kanji as 'asa' (morning), which creates a confusing contradiction with 'yoru' (night). Sample B also features slightly more natural phrasing.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B correctly pronounces 'ashita' (tomorrow), whereas Sample A incorrectly reads the kanji as 'asa' (morning), which creates a confusing contradiction with 'yoru' (night). Sample B also features slightly more natural phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a natural and accurate reading of the text. Sample B mispronounces 'ashita' (tomorrow) as 'asa' (morning) and has slightly more stilted prosody." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Incorrectly reads '明日' (ashita/asu) as 'asa' (morning), resulting in 'asa no yoru' (morning's night).", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日の夜' (ashita no yoru) as 'asa no yoru'.", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [] }, "limitations": [ "I am an AI and evaluate based on acoustic features and phonetic rules, which may differ from subjective human perception." ] } }, { "id": "m05-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-ko-KR-A.wav", "model": "s2.1-pro", "duration": 17.1102 }, "b": { "src": "audio/m05-ko-KR-B.wav", "model": "s2-pro", "duration": 18.1318 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples provide clear and intelligible speech with accurate text fidelity. The expressiveness is similar in both, with minor robotic pacing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples provide clear and intelligible speech with accurate text fidelity. The expressiveness is similar in both, with minor robotic pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately and are fully intelligible. They both exhibit slightly stiff, robotic prosody with unnatural pauses, making them comparable in overall quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing feels slightly mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The phrasing and pauses are somewhat rigid, lacking natural human cadence.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is a bit slow and mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and pauses between sentences feel slightly unnatural and stiff.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle prosodic nuances may be missed.", "I am not a native speaker of Korean, so subtle prosodic nuances may not be fully captured." ] } }, { "id": "m05-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-es-ES-A.wav", "model": "s2.1-pro", "duration": 16.5065 }, "b": { "src": "audio/m05-es-ES-B.wav", "model": "s2-pro", "duration": 18.6195 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a noticeable foreign accent. Sample A is preferred because its prosody is slightly more fluid and natural, whereas Sample B sounds more stilted and robotic.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a noticeable foreign accent. Sample A is preferred because its prosody is slightly more fluid and natural, whereas Sample B sounds more stilted and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text without reading the bracketed cues. However, both suffer from a strong non-native accent that detracts from the requested Spanish (Spain) locale, making them equally limited." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout the recording, lacking native es-ES phonetics.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent affecting consonants and vowels throughout.", "severity": "major", "timestamp_seconds": 1.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat stilted and disjointed, particularly around the time expression.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with English 'r' sounds (e.g., 'reservado', 'tarde').", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a strong foreign (English) accent, making it difficult to assess native es-ES nuances.", "Both samples exhibit a strong non-native (American English) accent, which affects the naturalness of the requested es-ES locale." ] } }, { "id": "m05-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.9956 }, "b": { "src": "audio/m05-fr-FR-B.wav", "model": "s2-pro", "duration": 16.2278 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples exhibit a heavy foreign accent that impacts naturalness. Sample A is slightly preferred for having marginally better phrasing and less robotic cadence than Sample B.", "confidence": 0.6, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples exhibit a heavy foreign accent that impacts naturalness. Sample A is slightly preferred for having marginally better phrasing and less robotic cadence than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.5, "rationale": "Both samples have a noticeable foreign accent, but Sample B's pronunciation is slightly more natural and intelligible than Sample A's." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent, though slightly better than A.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'biscuit' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is somewhat stiff and unnatural.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and rhythm.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native (English) accent, making it difficult to assess native French prosody.", "Both samples exhibit a strong non-native accent." ] } }, { "id": "m05-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-de-DE-A.wav", "model": "s2.1-pro", "duration": 18.3873 }, "b": { "src": "audio/m05-de-DE-B.wav", "model": "s2-pro", "duration": 17.7371 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have a very strong foreign accent. Sample A is slightly preferred for a warmer delivery that better matches the requested tone.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples have a very strong foreign accent. Sample A is slightly preferred for a warmer delivery that better matches the requested tone." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text but suffer from a noticeable foreign accent that impacts naturalness. Neither successfully captures the requested emotional shifts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong American accent throughout, affecting vowels and 'r' sounds.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on 'dir', 'reserviert', and 'reden'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and does not reflect the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong American accent throughout, affecting vowels and 'r' sounds.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on 'dir', 'reserviert', and 'reden'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery remains relatively flat, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong American accent, making it difficult to assess native-level prosody.", "Both samples exhibit a strong foreign (American) accent, affecting naturalness." ] } }, { "id": "m05-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.6442 }, "b": { "src": "audio/m05-pt-BR-B.wav", "model": "s2-pro", "duration": 19.2928 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples successfully deliver the target text with good intelligibility and audio quality. However, both suffer from a noticeable non-native accent and somewhat flat expressiveness, making a tie the most appropriate outcome.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text with good intelligibility and audio quality. However, both suffer from a noticeable non-native accent and somewhat flat expressiveness, making a tie the most appropriate outcome." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it has slightly better flow and more natural intonation. Both samples suffer from a noticeable non-native accent, but Sample A sounds more robotic and disjointed." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording, affecting vowel sounds and overall cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery remains relatively flat and does not clearly shift to a teasing tone at the end.", "severity": "minor", "timestamp_seconds": 13.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent affects several words like 'pressa' and 'janela'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Exhibits a non-native accent that affects the natural flow of Brazilian Portuguese.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warmth and teasing shifts, sounding somewhat robotic.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is disjointed and lacks natural phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects vowels and rhythm throughout.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent for Brazilian Portuguese.", "Both samples exhibit a strong non-native accent for Brazilian Portuguese." ] } }, { "id": "m05-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m05-ar-A.wav", "model": "s2-pro", "duration": 20.5699 }, "b": { "src": "audio/m05-ar-B.wav", "model": "s2.1-pro", "duration": 19.1071 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit robotic prosody and unnatural pauses, particularly before 'النافذة'. Neither successfully captures the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit robotic prosody and unnatural pauses, particularly before 'النافذة'. Neither successfully captures the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly better naturalness and flow compared to Sample B, which sounds a bit more disjointed. Both convey the text accurately." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed phrasing and unnatural pauses.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing in the middle sentence.", "severity": "minor", "timestamp_seconds": 6.0 } ] }, "limitations": [ "Evaluation of Arabic prosody and naturalness is based on general MSA standards.", "Evaluation of Arabic prosody and regional accent nuances is limited." ] } } ] }, { "id": "m06", "name": "Jacob Winchester", "slug": "jacob-winchester", "gender": "Male", "rank": 6, "portrait": "portraits/m06.webp", "language": "8 languages", "popularity": { "callers": 13, "calls": 15 }, "pairs": [ { "id": "m06-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-zh-CN-A.wav", "model": "s2-pro", "duration": 22.3579 }, "b": { "src": "audio/m06-zh-CN-B.wav", "model": "s2.1-pro", "duration": 15.6705 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is preferred due to significantly better audio quality. Sample A suffers from severe, continuous static noise that disrupts listening.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B is preferred due to significantly better audio quality. Sample A suffers from severe, continuous static noise that disrupts listening." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a much cleaner listening experience. Sample B suffers from severe bursts of static and distortion between phrases, significantly degrading audio quality." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 2 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Loud, continuous static noise throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Loud bursts of static and noise between spoken phrases.", "severity": "major", "timestamp_seconds": 2.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is relatively flat, lacking the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is slightly unnatural and stiff.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native speaker of all regional Mandarin accents." ] } }, { "id": "m06-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.0374 }, "b": { "src": "audio/m06-ja-JP-B.wav", "model": "s2-pro", "duration": 17.412 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer audio and correct pronunciation. Sample A mispronounces the first word and has a slightly muffled audio quality.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides clearer audio and correct pronunciation. Sample A mispronounces the first word and has a slightly muffled audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A features more natural Japanese pitch accent and phrasing. Sample B has minor pitch accent unnaturalness on a few words, making A the better overall performance." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (Isoganakute) as something like 'Izaganakute'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "other", "evidence": "Audio sounds slightly muffled with background noise.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Slightly unnatural pitch accent on '急がなくて' and '約束'.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "other", "evidence": "Simulated background noise present throughout.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "BA", "category": "other", "evidence": "Noticeable simulated background noise throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 } ] }, "limitations": [ "I am not a native Japanese speaker, but the pronunciation error in Sample A is distinct." ] } }, { "id": "m06-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-ko-KR-A.wav", "model": "s2-pro", "duration": 16.0885 }, "b": { "src": "audio/m06-ko-KR-B.wav", "model": "s2.1-pro", "duration": 16.9708 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers clearer audio quality and slightly more natural prosody. Sample B suffers from noticeable background noise and flatter delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers clearer audio quality and slightly more natural prosody. Sample B suffers from noticeable background noise and flatter delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers smoother phrasing and better overall audio quality. Sample A has noticeable background noise and slightly choppy transitions." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing between sentences.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is subtle and could be more pronounced.", "severity": "minor", "timestamp_seconds": 11.0 } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background noise and hissing throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery remains relatively flat, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly choppy transitions and unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss throughout the recording.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluated by a non-native speaker; focused on general prosody, clarity, and audio quality." ] } }, { "id": "m06-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.6256 }, "b": { "src": "audio/m06-es-ES-B.wav", "model": "s2-pro", "duration": 15.09 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe English accent and robotic prosody, making them equally unnatural for the target locale. Neither successfully conveys the requested warm or teasing delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent and robotic prosody, making them equally unnatural for the target locale. Neither successfully conveys the requested warm or teasing delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent and identical mispronunciations. Sample A is preferred because it lacks the background noise present in Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 2.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, distorting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted, unnatural cadence with inappropriate pauses.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'tengas' pronounced as 'tangas', 'tranquilo' as 'tranquila'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background noise/static throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and lacks the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, distorting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted, unnatural cadence with inappropriate pauses.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'tengas' pronounced as 'tangas', 'tranquilo' as 'tranquila'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and lacks the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong non-native (English) accent.", "Both samples exhibit a very strong non-native accent." ] } }, { "id": "m06-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-fr-FR-A.wav", "model": "s2.1-pro", "duration": 14.0916 }, "b": { "src": "audio/m06-fr-FR-B.wav", "model": "s2-pro", "duration": 14.1845 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a severe English accent when reading French, making them equally unnatural and difficult to understand. Background noise is also present in both.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a severe English accent when reading French, making them equally unnatural and difficult to understand. Background noise is also present in both." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy foreign accent with numerous pronunciation errors. Neither sounds like a native French speaker, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on 'Prends ton temps'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronunciation of 'deux places'.", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and noise throughout.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavily anglicized pronunciation throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on 'Prends ton temps'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronunciation of 'deux places'.", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and noise throughout.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavily anglicized pronunciation throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Cannot determine if the background noise is intentional room tone or an artifact.", "Both samples feature a very strong non-native (likely American) accent, making it difficult to assess native French prosody." ] } }, { "id": "m06-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-de-DE-A.wav", "model": "s2.1-pro", "duration": 14.672 }, "b": { "src": "audio/m06-de-DE-B.wav", "model": "s2-pro", "duration": 14.4166 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a heavy foreign accent that impacts pronunciation and naturalness. Sample B has slightly better flow and expressiveness compared to the more disjointed delivery in Sample A.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a heavy foreign accent that impacts pronunciation and naturalness. Sample B has slightly better flow and expressiveness compared to the more disjointed delivery in Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong English accent and mispronounce 'Keks' as the English word 'cakes'. They are very similar in quality and defects." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 13.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted and disjointed phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 12.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 12.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 12.5 } ] }, "limitations": [ "Both samples feature a very strong foreign (American) accent, affecting pronunciation of German words.", "Both samples exhibit a strong non-native accent." ] } }, { "id": "m06-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.7603 }, "b": { "src": "audio/m06-pt-BR-B.wav", "model": "s2-pro", "duration": 16.5297 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B has slightly better prosody and flow compared to Sample A, which sounds more disjointed. Both samples feature a non-native accent, but B is more natural overall.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B has slightly better prosody and flow compared to Sample A, which sounds more disjointed. Both samples feature a non-native accent, but B is more natural overall." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A offers significantly better audio quality and naturalness. Sample B suffers from noticeable reverb and robotic prosody, making it less pleasant to listen to." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is somewhat disjointed and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable reverb and background noise throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural rhythm and phrasing.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 16.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 15.0 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent when speaking Brazilian Portuguese." ] } }, { "id": "m06-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m06-ar-A.wav", "model": "s2.1-pro", "duration": 19.3625 }, "b": { "src": "audio/m06-ar-B.wav", "model": "s2-pro", "duration": 19.316 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to better text fidelity and slightly smoother prosody. Sample B mispronounces a verb, changing the meaning from 'I booked' to 'she booked', and has more distracting noise gating.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to better text fidelity and slightly smoother prosody. Sample B mispronounces a verb, changing the meaning from 'I booked' to 'she booked', and has more distracting noise gating." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because Sample A mispronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked), changing the meaning. Both suffer from unnatural pauses and poor audio quality." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and pauses throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss present during speech.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and low audio quality.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Heard 'Hajazat' (she booked) instead of expected 'Hajaztu' (I booked).", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "distortion", "evidence": "Distracting noise gating and background hiss.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy delivery with unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "text_substitution", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and low audio quality.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit background hiss and somewhat robotic prosody.", "I am not a native speaker of Modern Standard Arabic, but can identify basic pronunciation and word errors." ] } } ] }, { "id": "m07", "name": "Dominic Vance", "slug": "dominic-vance-39abf1", "gender": "Male", "rank": 7, "portrait": "portraits/m07.webp", "language": "8 languages", "popularity": { "callers": 11, "calls": 19 }, "pairs": [ { "id": "m07-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-zh-CN-A.wav", "model": "s2-pro", "duration": 21.313 }, "b": { "src": "audio/m07-zh-CN-B.wav", "model": "s2.1-pro", "duration": 18.4105 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples feature a strong foreign accent with inaccurate tones. Neither sounds like a native Mandarin speaker, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong foreign accent with inaccurate tones. Neither sounds like a native Mandarin speaker, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent and incorrect Mandarin tones throughout. Neither is suitable for native-level applications, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent with incorrect tones throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent and incorrect tones throughout the entire utterance.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent with incorrect tones throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent and incorrect tones throughout the entire utterance.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making tone evaluation challenging.", "Both samples exhibit a very strong foreign accent with pervasive tone errors, making native-level evaluation difficult." ] } }, { "id": "m07-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-ja-JP-A.wav", "model": "s2-pro", "duration": 20.9183 }, "b": { "src": "audio/m07-ja-JP-B.wav", "model": "s2.1-pro", "duration": 21.3362 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A better captures the requested 'warm, reassuring' and 'gently teasing' delivery. Sample B sounds too forceful and loud for the intended tone.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A better captures the requested 'warm, reassuring' and 'gently teasing' delivery. Sample B sounds too forceful and loud for the intended tone." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent. Sample B is preferred for its slightly better phrasing and pacing, whereas Sample A is more choppy and exaggerated." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects pitch accent and vowel length throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Noticeable non-native accent affecting the natural flow of Japanese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is too loud and forceful for a 'warm, reassuring' tone.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects pitch accent and vowel length throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Heavy non-native accent and choppy phrasing throughout the sample.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, which affects the naturalness of the Japanese pitch accent and intonation.", "Both samples feature a strong non-native accent, which affects naturalness and prosody." ] } }, { "id": "m07-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-ko-KR-A.wav", "model": "s2-pro", "duration": 19.8501 }, "b": { "src": "audio/m07-ko-KR-B.wav", "model": "s2.1-pro", "duration": 21.8702 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural flow and better phrasing. Sample B suffers from choppy, disjointed prosody that disrupts the delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural flow and better phrasing. Sample B suffers from choppy, disjointed prosody that disrupts the delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural cadence and phrasing. Sample A suffers from excessively long and unnatural pauses between sentences, disrupting the flow." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy and disjointed phrasing throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Excessively long and unnatural pauses between sentences.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, but can assess general prosody and intelligibility." ] } }, { "id": "m07-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-es-ES-A.wav", "model": "s2.1-pro", "duration": 19.2464 }, "b": { "src": "audio/m07-es-ES-B.wav", "model": "s2-pro", "duration": 19.4322 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to produce a convincing Spanish (Spain) accent, suffering from severe English-accented mispronunciations. Sample A is marginally better as it attempts some Spanish phonemes, whereas Sample B reads the text almost entirely with English pronunciation rules.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to produce a convincing Spanish (Spain) accent, suffering from severe English-accented mispronunciations. Sample A is marginally better as it attempts some Spanish phonemes, whereas Sample B reads the text almost entirely with English pronunciation rules." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a heavy non-native accent. Sample A is preferred because its tone is closer to the requested warm delivery, whereas Sample B sounds overly gruff and aggressive." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent throughout, e.g., 'reservado' and 'galleta' are pronounced with English phonetics.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is disjointed and lacks natural Spanish cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, distorting Spanish phonemes.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice sounds gruff and aggressive, missing the requested 'warm, reassuring' tone.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Extreme English accent applied to Spanish words, such as pronouncing 'galleta' with a hard 'l' sound.", "severity": "blocking", "timestamp_seconds": 17.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones, sounding like a flat reading.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., English 'r' sounds in 'reservado'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit severe foreign accents, sounding like English speakers reading Spanish text.", "Both samples feature a strong non-native (English-like) accent that affects the pronunciation of Spanish phonemes." ] } }, { "id": "m07-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-fr-FR-A.wav", "model": "s2.1-pro", "duration": 17.7603 }, "b": { "src": "audio/m07-fr-FR-B.wav", "model": "s2-pro", "duration": 17.2727 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe English accent, mispronouncing most French phonemes (especially nasal vowels and the 'eu' sound). Neither sounds like a native French speaker, making them equally limited.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe English accent, mispronouncing most French phonemes (especially nasal vowels and the 'eu' sound). Neither sounds like a native French speaker, making them equally limited." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent when speaking French, making them equally unnatural. There is no clear winner as both fail to produce native-sounding French." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'temps' pronounced as 'tam', 'deux' as 'doo'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout the French text.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'temps' pronounced as 'tam', 'deux' as 'doo'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout the French text.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a very strong English accent, making the French pronunciation highly unnatural." ] } }, { "id": "m07-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-de-DE-A.wav", "model": "s2.1-pro", "duration": 18.9445 }, "b": { "src": "audio/m07-de-DE-B.wav", "model": "s2-pro", "duration": 20.8021 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better naturalness and audio quality. Sample A has a more robotic cadence and noticeable synthetic artifacts.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and audio quality. Sample A has a more robotic cadence and noticeable synthetic artifacts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A successfully captures the warm and reassuring tone requested. Sample B sounds aggressive and rushed, completely missing the intended delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation sound slightly mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is aggressive and rushed, failing to sound warm or gently teasing.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [] }, "limitations": [ "Evaluator is not a native German speaker, but can assess general pronunciation and prosody." ] } }, { "id": "m07-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-pt-BR-A.wav", "model": "s2-pro", "duration": 20.9647 }, "b": { "src": "audio/m07-pt-BR-B.wav", "model": "s2.1-pro", "duration": 22.4275 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from a heavy foreign accent and stilted delivery. Sample A is slightly preferred for having marginally smoother phrasing and less choppy cadence than Sample B.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a heavy foreign accent and stilted delivery. Sample A is slightly preferred for having marginally smoother phrasing and less choppy cadence than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a severe foreign accent that degrades naturalness. Sample B is slightly preferred as its phrasing and flow are marginally smoother than Sample A's robotic delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is somewhat stilted and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent affects the pronunciation of most words.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is very choppy and disjointed.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, mispronouncing vowels and consonants (e.g., 'amanhã', 'lugares').", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed and unnatural cadence with awkward pauses.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody.", "Both samples exhibit a very strong foreign accent, making it difficult to assess native-level prosody." ] } }, { "id": "m07-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m07-ar-A.wav", "model": "s2.1-pro", "duration": 22.9384 }, "b": { "src": "audio/m07-ar-B.wav", "model": "s2-pro", "duration": 22.7758 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from robotic cadence and unnatural pauses. Sample B is preferred slightly due to better audio quality and fewer background artifacts.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from robotic cadence and unnatural pauses. Sample B is preferred slightly due to better audio quality and fewer background artifacts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A has better pacing and audio quality than Sample B, which suffers from unnatural mid-sentence pauses and noticeable distortion. Both lack natural expressiveness." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural phrasing and pauses, such as before 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural mid-sentence pauses, such as between 'qurba' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 9.0 }, { "order": "BA", "category": "distortion", "evidence": "The voice has a noticeable distorted, synthetic quality throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The tone is monotone and does not match the requested emotional cues.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted cadence and unnatural pauses, such as before 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat disjointed and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery does not effectively convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both models exhibit robotic prosody and unnatural pauses.", "Both models read the time in a slightly colloquial manner rather than strict MSA.", "Arabic pronunciation and prosody evaluation." ] } } ] }, { "id": "m08", "name": "Argos", "slug": "argos-763339", "gender": "Male", "rank": 8, "portrait": "portraits/m08.webp", "language": "8 languages", "popularity": { "callers": 11, "calls": 12 }, "pairs": [ { "id": "m08-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-zh-CN-A.wav", "model": "s2-pro", "duration": 16.7154 }, "b": { "src": "audio/m08-zh-CN-B.wav", "model": "s2.1-pro", "duration": 15.5312 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly better pacing and a warmer tone that better fits the prompt. Sample B sounds more rushed and robotic. Both suffer from muffled audio quality.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly better pacing and a warmer tone that better fits the prompt. Sample B sounds more rushed and robotic. Both suffer from muffled audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides clearer audio quality and more natural phrasing. It captures the intended warmth and teasing tone better than Sample A, which sounds slightly muffled." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is somewhat stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery feels rushed and mechanical, lacking the requested warmth.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Slightly muffled audio quality with noticeable background hiss.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m08-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-ja-JP-A.wav", "model": "s2-pro", "duration": 19.1071 }, "b": { "src": "audio/m08-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.6179 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A reads the text accurately. Sample B has clear text fidelity errors, mispronouncing 'isoganakute' and 'ashita'. Both samples have a noticeable non-native accent.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A reads the text accurately. Sample B has clear text fidelity errors, mispronouncing 'isoganakute' and 'ashita'. Both samples have a noticeable non-native accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides slightly better expressiveness by including a chuckle that fits the teasing direction. Sample A adds an unscripted word." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, affecting prosody and vowel lengths.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Intonation and pitch accents sound unnatural and non-native throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Heard 'isaganakute' instead of expected 'isoganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "text_substitution", "evidence": "Heard 'asa no yoru' instead of expected 'ashita no yoru'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "text_addition", "evidence": "Added the word 'うん' (un) before 'でも'.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Intonation and pitch accents sound unnatural and non-native throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is assessing Japanese pronunciation and prosody as a non-native speaker, focusing on clear phoneme errors.", "Both samples exhibit non-native prosody and intonation patterns in Japanese." ] } }, { "id": "m08-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-ko-KR-A.wav", "model": "s2-pro", "duration": 16.9476 }, "b": { "src": "audio/m08-ko-KR-B.wav", "model": "s2.1-pro", "duration": 16.3671 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural and expressive delivery that better captures the requested warm tone. Sample B sounds slightly more robotic and stiff in its phrasing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better captures the requested warm tone. Sample B sounds slightly more robotic and stiff in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery that better captures the warm and reassuring tone requested. Sample A sounds slightly more robotic and disjointed in its pacing." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff phrasing throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat disjointed and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle accent or intonation nuances may not be fully evaluated.", "I am not a native Korean speaker, but I can evaluate general prosody and text fidelity." ] } }, { "id": "m08-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-es-ES-A.wav", "model": "s2-pro", "duration": 16.7154 }, "b": { "src": "audio/m08-es-ES-B.wav", "model": "s2.1-pro", "duration": 14.9739 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is preferred for its more fluid pacing and natural phrasing. Sample A suffers from overly long pauses and a more robotic cadence.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred for its more fluid pacing and natural phrasing. Sample A suffers from overly long pauses and a more robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy non-native (likely French) accent that severely impacts the requested es-ES locale. Neither successfully conveys the requested emotional shifts." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnaturally long pauses between sentences and a stiff cadence.", "severity": "major", "timestamp_seconds": 6.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent affecting vowels and consonants (e.g., 'tu día', 'tranquilo').", "severity": "minor", "timestamp_seconds": 10.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affecting vowels and consonants (e.g., 'tu día', 'galleta').", "severity": "major", "timestamp_seconds": 9.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the warm or gently teasing tones requested.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent affecting vowels and consonants.", "severity": "minor", "timestamp_seconds": 10.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent on words like 'reservado' and 'galleta'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stilted pacing and unnatural pauses between phrases.", "severity": "major", "timestamp_seconds": 6.0 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native (possibly French) accent, affecting the naturalness of the Spanish (Spain) locale.", "Both samples exhibit a strong non-native accent, making locale-specific evaluation difficult." ] } }, { "id": "m08-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-fr-FR-A.wav", "model": "s2-pro", "duration": 14.347 }, "b": { "src": "audio/m08-fr-FR-B.wav", "model": "s2.1-pro", "duration": 12.8609 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural conversational flow and pacing. Sample A has slightly longer pauses that feel less organic.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural conversational flow and pacing. Sample A has slightly longer pauses that feel less organic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar unnatural pauses and slightly muffled audio quality. Neither clearly outperforms the other in expressiveness or naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pauses between sentences feel slightly unnatural and rigid.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'fenêtre' and 'parler'.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'fenêtre' and 'parler'.", "severity": "minor", "timestamp_seconds": 6.5 } ] }, "limitations": [ "I cannot evaluate the exact regional accent accuracy beyond general French intelligibility." ] } }, { "id": "m08-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-de-DE-A.wav", "model": "s2.1-pro", "duration": 18.6427 }, "b": { "src": "audio/m08-de-DE-B.wav", "model": "s2-pro", "duration": 18.2944 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from identical pronunciation errors and highly robotic prosody. Neither successfully conveys the requested delivery, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from identical pronunciation errors and highly robotic prosody. Neither successfully conveys the requested delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples share the same pronunciation errors for 'Zeit' and 'versprich'. Sample A is preferred slightly due to a cleaner audio environment with less background noise." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'Zeit' with a /z/ sound instead of /ts/.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is disjointed with unnatural pauses and flat intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Zeit' as 'Seit' with a soft 's'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Unnatural stress and pause in 'versprich'.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "distortion", "evidence": "Slight room echo or background noise present.", "severity": "minor", "timestamp_seconds": 2.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'Zeit' with a /z/ sound instead of /ts/.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is unnatural with awkward pauses between phrases.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Zeit' as 'Seit' with a soft 's'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Unnatural stress and pause in 'versprich'.", "severity": "minor", "timestamp_seconds": 14.0 } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, making it difficult to assess native-level nuances.", "Both samples exhibit identical pronunciation errors, suggesting a shared underlying model limitation with German phonetics." ] } }, { "id": "m08-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.8764 }, "b": { "src": "audio/m08-pt-BR-B.wav", "model": "s2-pro", "duration": 18.1318 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to use the requested Brazilian Portuguese locale, instead sounding like European Portuguese. Sample B is slightly preferred due to a smoother, more natural cadence, whereas A sounds more disjointed.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, instead sounding like European Portuguese. Sample B is slightly preferred due to a smoother, more natural cadence, whereas A sounds more disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese accent, defaulting to European Portuguese instead. They are otherwise comparable in intelligibility and text fidelity." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is somewhat disjointed with unnatural pauses between clauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Speaker uses a European Portuguese accent instead of the requested pt-BR locale.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Speaker uses a European Portuguese accent (e.g., pronunciation of 'dia', 'lugares') instead of the requested pt-BR locale.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a European Portuguese accent rather than the requested Brazilian Portuguese.", "Both samples use a European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "m08-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m08-ar-A.wav", "model": "s2.1-pro", "duration": 22.6365 }, "b": { "src": "audio/m08-ar-B.wav", "model": "s2-pro", "duration": 22.7062 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B is preferred due to slightly better intelligibility and more natural prosody. Sample A has some minor pronunciation issues and a slightly more robotic cadence.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred due to slightly better intelligibility and more natural prosody. Sample A has some minor pronunciation issues and a slightly more robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a major text fidelity issue, reading the time '7:45' in French rather than Arabic. They are tied due to this shared defect." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Minor pronunciation errors in the middle section.", "severity": "minor", "timestamp_seconds": 6.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and phrasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Reads '7:45' as French 'sept quarante-cinq'.", "severity": "major", "timestamp_seconds": 5.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Minor pronunciation errors in the middle section.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "wrong_language", "evidence": "Reads '7:45' as French 'sept quarante-cinq'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'قطعة' (piece) as 'قطرة' (drop).", "severity": "major", "timestamp_seconds": 18.5 } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, so my evaluation of pronunciation and prosody is based on general phonetic principles and may miss subtle nuances.", "Both samples read the time in French instead of Arabic." ] } } ] }, { "id": "m09", "name": "Luca Castellano", "slug": "luca-castellano-6ec81a", "gender": "Male", "rank": 9, "portrait": "portraits/m09.webp", "language": "8 languages", "popularity": { "callers": 10, "calls": 21 }, "pairs": [ { "id": "m09-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-zh-CN-A.wav", "model": "s2.1-pro", "duration": 14.3005 }, "b": { "src": "audio/m09-zh-CN-B.wav", "model": "s2-pro", "duration": 14.9042 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B sounds slightly more rigid.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B sounds slightly more rigid." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly more natural phrasing and pacing, making it marginally better, though both struggle to convey the requested emotional shifts." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "Emotional shifts are subtle and do not fully capture the teasing tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff delivery throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery remains relatively flat, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m09-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-ja-JP-A.wav", "model": "s2.1-pro", "duration": 16.5761 }, "b": { "src": "audio/m09-ja-JP-B.wav", "model": "s2-pro", "duration": 17.2263 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A delivers a more natural and expressive performance that better matches the requested warm and teasing tones. Sample B sounds slightly more robotic and flat.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A delivers a more natural and expressive performance that better matches the requested warm and teasing tones. Sample B sounds slightly more robotic and flat." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and follow the text accurately. However, both exhibit slightly robotic prosody and limited expressiveness, making it a tie." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat unnatural and stiff.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery feels slightly stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Japanese speaker, but I can evaluate general prosody and intelligibility.", "I am not a native speaker of Japanese, so subtle prosodic nuances may not be fully captured." ] } }, { "id": "m09-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-ko-KR-A.wav", "model": "s2.1-pro", "duration": 15.4615 }, "b": { "src": "audio/m09-ko-KR-B.wav", "model": "s2-pro", "duration": 17.1798 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural delivery and effectively captures the requested emotional shift, including a subtle chuckle for the teasing section. Sample B is slightly more rigid in its prosody.", "confidence": 0.825, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.85, "rationale": "Sample A provides a more natural delivery and effectively captures the requested emotional shift, including a subtle chuckle for the teasing section. Sample B is slightly more rigid in its prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural delivery and better captures the requested shift to gentle teasing, including a subtle chuckle. Sample A is slightly more robotic in its pacing." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff and lacks the requested emotional shift.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing and pauses feel slightly unnatural and mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not very pronounced.", "severity": "minor", "timestamp_seconds": 11.5 } ] }, "limitations": [] } }, { "id": "m09-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.9275 }, "b": { "src": "audio/m09-es-ES-B.wav", "model": "s2-pro", "duration": 15.2061 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are intelligible and follow the text accurately, but both suffer from a noticeable non-native accent and somewhat robotic prosody. Neither clearly outperforms the other.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples are intelligible and follow the text accurately, but both suffer from a noticeable non-native accent and somewhat robotic prosody. Neither clearly outperforms the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a noticeable non-native accent and flat delivery that misses the requested emotional cues. They are equally intelligible and preserve the text." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Non-native accent on words like 'reservado'.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Non-native accent throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a non-native accent for Spain Spanish.", "Both samples exhibit strong non-native accents, making it difficult to assess native-level prosody." ] } }, { "id": "m09-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-fr-FR-A.wav", "model": "s2-pro", "duration": 13.7665 }, "b": { "src": "audio/m09-fr-FR-B.wav", "model": "s2.1-pro", "duration": 13.2324 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a very strong foreign accent, mispronouncing several French words (e.g., 'biscuit', 'tranquille'). Neither is clearly better as both sound like an English speaker reading French phonetically.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a very strong foreign accent, mispronouncing several French words (e.g., 'biscuit', 'tranquille'). Neither is clearly better as both sound like an English speaker reading French phonetically." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a heavy non-native accent and choppy prosody, making them equally unnatural for the requested French locale. There is no clear winner." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American accent throughout, mispronouncing words like 'deux', 'tranquille', and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural phrasing for French.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American accent throughout, mispronouncing words like 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural phrasing for French.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a severe American English accent applied to French text, impacting naturalness and intelligibility.", "Both samples exhibit a very strong non-native accent." ] } }, { "id": "m09-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-de-DE-A.wav", "model": "s2-pro", "duration": 15.833 }, "b": { "src": "audio/m09-de-DE-B.wav", "model": "s2.1-pro", "duration": 15.4848 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples have a noticeable foreign accent, but Sample B is preferred because Sample A mispronounces a key word.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a noticeable foreign accent, but Sample B is preferred because Sample A mispronounces a key word." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a strong foreign accent that impacts naturalness. Sample A is preferred because Sample B significantly mispronounces 'reserviert'." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'reserviert' as 'resiert'.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'reserviert' is mispronounced, missing the 'v' sound.", "severity": "major", "timestamp_seconds": 4.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'Lass dir Zeit' sounds like 'Last year tsight'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a strong non-native accent.", "Both samples feature a very strong English accent, limiting native-like naturalness." ] } }, { "id": "m09-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-pt-BR-A.wav", "model": "s2.1-pro", "duration": 16.5761 }, "b": { "src": "audio/m09-pt-BR-B.wav", "model": "s2-pro", "duration": 17.4352 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A has a more natural flow and better prosody than Sample B, which sounds choppy and robotic. Both exhibit non-native accents, but A is more listenable.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A has a more natural flow and better prosody than Sample B, which sounds choppy and robotic. Both exhibit non-native accents, but A is more listenable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text with accurate wording and clear intelligibility. However, both suffer from robotic prosody and fail to convey the requested warm and teasing emotions." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, affecting vowel sounds and rhythm.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and misses the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech sounds mechanical and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The tone remains neutral throughout, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech is choppy with unnatural pauses and cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects the naturalness of the Brazilian Portuguese.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm or teasing emotions requested.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested warm and teasing emotional shifts.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the warm, reassuring, and gently teasing tones requested in the prompt.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit non-native accents for Brazilian Portuguese.", "Both samples exhibit a noticeable non-native or robotic accent, making it difficult to assess true native Brazilian Portuguese prosody." ] } }, { "id": "m09-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m09-ar-A.wav", "model": "s2-pro", "duration": 18.3176 }, "b": { "src": "audio/m09-ar-B.wav", "model": "s2.1-pro", "duration": 16.1349 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B correctly pronounces 'حجزت' as 'hajaztu' (I booked), fitting the context, whereas Sample A pronounces it as 'hajazat' (she booked). Both suffer from unnatural pauses, but B has better text fidelity.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B correctly pronounces 'حجزت' as 'hajaztu' (I booked), fitting the context, whereas Sample A pronounces it as 'hajazat' (she booked). Both suffer from unnatural pauses, but B has better text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A correctly vowelizes the verb to mean 'I booked', which fits the context. Sample B vowelizes it as 'she booked', altering the intended meaning. Both have slightly unnatural pauses." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked), changing the meaning.", "severity": "major", "timestamp_seconds": 1.8 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafidha'.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafidha'.", "severity": "minor", "timestamp_seconds": 5.5 } ] }, "limitations": [ "I am not a native speaker of Arabic, but can identify basic pronunciation and prosody issues.", "I am evaluating the Arabic pronunciation and prosody based on standard rules, but regional variations in number reading may exist." ] } } ] }, { "id": "m10", "name": "I want a dark but clingy and possessive mafia boss", "slug": "i-want-a-dark-but-clingy-and-possessive-mafia-boss-b9f5a1", "gender": "Male", "rank": 10, "portrait": "portraits/m10.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 17 }, "pairs": [ { "id": "m10-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-zh-CN-A.wav", "model": "s2-pro", "duration": 15.09 }, "b": { "src": "audio/m10-zh-CN-B.wav", "model": "s2.1-pro", "duration": 14.7417 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a much more natural and expressive delivery that aligns well with the requested tone. Sample B sounds robotic and lacks the intended warmth and teasing.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a much more natural and expressive delivery that aligns well with the requested tone. Sample B sounds robotic and lacks the intended warmth and teasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. However, both struggle to convey the requested emotional shifts, resulting in a tie." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to clearly distinguish between the warm and teasing tones.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Flat intonation and unnatural pacing throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and gently teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m10-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-ja-JP-A.wav", "model": "s2-pro", "duration": 17.2495 }, "b": { "src": "audio/m10-ja-JP-B.wav", "model": "s2.1-pro", "duration": 17.5978 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A sounds much more natural with appropriate Japanese pitch accent and intonation. Sample B has a noticeable foreign accent and unnatural prosody that detracts from the delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A sounds much more natural with appropriate Japanese pitch accent and intonation. Sample B has a noticeable foreign accent and unnatural prosody that detracts from the delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly more natural phrasing and pacing. Sample A sounds a bit more disjointed, though both lack strong expressiveness." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is somewhat neutral, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pitch accent and intonation throughout the recording, sounding like a non-native speaker.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pauses and pacing feel slightly unnatural and disjointed.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Japanese speaker, so subtle nuances in intonation might be missed." ] } }, { "id": "m10-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-ko-KR-A.wav", "model": "s2.1-pro", "duration": 15.2061 }, "b": { "src": "audio/m10-ko-KR-B.wav", "model": "s2-pro", "duration": 17.2495 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are intelligible and accurate to the text but lack the requested emotional shifts. They sound equally flat and slightly robotic, making it a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text but lack the requested emotional shifts. They sound equally flat and slightly robotic, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both suffer from robotic prosody and fail to convey the requested emotional shifts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is generally flat and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and gently teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Speech sounds slightly unnatural and mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Does not effectively convey the shift to gentle teasing.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation sound somewhat mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not capture the warm or teasing directions.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation feel somewhat mechanical throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to clearly distinguish between the warm and teasing tones requested.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a similarly flat, somewhat robotic delivery.", "Both samples exhibit similar robotic prosody, making it difficult to strongly prefer one over the other." ] } }, { "id": "m10-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-es-ES-A.wav", "model": "s2-pro", "duration": 15.3919 }, "b": { "src": "audio/m10-es-ES-B.wav", "model": "s2.1-pro", "duration": 14.556 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B delivers the text with a natural Spanish accent and appropriate warmth. Sample A has a strong foreign accent that makes the delivery sound unnatural.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B delivers the text with a natural Spanish accent and appropriate warmth. Sample A has a strong foreign accent that makes the delivery sound unnatural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent that heavily impacts naturalness and pronunciation. Neither successfully captures a native Spanish delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, particularly noticeable on 'reservado', 'tarde', and 'tranquilo'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'prisa' and 'galleta'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong foreign (English) accent, making it difficult to assess native es-ES prosody." ] } }, { "id": "m10-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-fr-FR-A.wav", "model": "s2-pro", "duration": 14.0219 }, "b": { "src": "audio/m10-fr-FR-B.wav", "model": "s2.1-pro", "duration": 13.3718 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a slightly more natural delivery with better pacing and expressiveness. Sample B sounds a bit more robotic and rushed in its phrasing.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a slightly more natural delivery with better pacing and expressiveness. Sample B sounds a bit more robotic and rushed in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly smoother pacing and naturalness compared to Sample A. However, both samples are quite robotic and fail to convey the requested warmth and teasing." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is somewhat stiff and lacks dynamic intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Does not effectively shift between warm and teasing as requested.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing feels slightly rushed and less natural compared to A.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and disjointed, lacking natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, and gently teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am evaluating based on the provided audio and text, without knowledge of the specific model or reference voice." ] } }, { "id": "m10-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-de-DE-A.wav", "model": "s2-pro", "duration": 15.9492 }, "b": { "src": "audio/m10-de-DE-B.wav", "model": "s2.1-pro", "duration": 16.2278 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a noticeable non-native accent and fail to convey the requested warm and teasing emotions, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a noticeable non-native accent and fail to convey the requested warm and teasing emotions, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural German pronunciation. Sample B exhibits a noticeable foreign accent that detracts from the overall naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "English-sounding 'r' in 'reserviert' and 'reden'.", "severity": "minor", "timestamp_seconds": 4.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign accent on words like 'reden' and 'reserviert'.", "severity": "minor", "timestamp_seconds": 8.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Slight non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing throughout the utterance.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Cannot evaluate subtle regional German accent variations.", "Evaluation of subtle emotional shifts may be subjective." ] } }, { "id": "m10-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-pt-BR-A.wav", "model": "s2-pro", "duration": 16.9476 }, "b": { "src": "audio/m10-pt-BR-B.wav", "model": "s2.1-pro", "duration": 16.6457 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from unnatural, robotic prosody and fail to convey the requested emotions. Sample B is preferred slightly because Sample A includes a text fidelity error.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from unnatural, robotic prosody and fail to convey the requested emotions. Sample B is preferred slightly because Sample A includes a text fidelity error." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred because it avoids the text error found in Sample B and is slightly closer to the requested Brazilian Portuguese locale. Sample B has a distinct European Portuguese accent and adds an incorrect plural." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_addition", "evidence": "Adds an 's' to 'noite', saying 'noites'.", "severity": "minor", "timestamp_seconds": 5.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and does not sound warm, reassuring, or teasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The speech has a strong non-native accent and unnatural cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Heard 'noites' instead of the expected 'noite'.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The accent is distinctly European Portuguese rather than the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is completely flat, missing the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The voice sounds mechanical with unnatural phrasing and a non-native accent.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff and lacks the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit strong non-native accents and robotic prosody, making it difficult to assess regional authenticity.", "Both samples exhibit non-native or mixed accent traits for the requested Brazilian Portuguese locale." ] } }, { "id": "m10-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m10-ar-A.wav", "model": "s2-pro", "duration": 18.2015 }, "b": { "src": "audio/m10-ar-B.wav", "model": "s2.1-pro", "duration": 17.8996 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit similar minor pronunciation issues and slightly robotic prosody. Neither clearly outperforms the other in naturalness or expressiveness.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar minor pronunciation issues and slightly robotic prosody. Neither clearly outperforms the other in naturalness or expressiveness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody, disjointed phrasing, and lack of natural flow. Neither successfully captures the requested warm or teasing delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'مقعدين' with a 'k' sound instead of 'q'.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses and cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is choppy and lacks natural intonation or fluid transitions between words.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_omission", "evidence": "Pronounces 'للاستعجال' as 'لاستعجال'.", "severity": "minor", "timestamp_seconds": 0.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'مقعدين' with a 'k' sound instead of 'q'.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses and cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is highly unnatural with disjointed pauses between words throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_omission", "evidence": "Pronounces 'للاستعجال' as 'لاستعجال', omitting the definite article sound.", "severity": "minor", "timestamp_seconds": 0.5 } ] }, "limitations": [ "I am not a native Arabic speaker, so subtle prosodic nuances might be missed.", "Both samples exhibit highly robotic and disjointed prosody, making it difficult to distinguish subtle expressiveness." ] } } ] }, { "id": "m11", "name": "Dante Moretti", "slug": "dante-moretti-fb1fd8", "gender": "Male", "rank": 11, "portrait": "portraits/m11.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 11 }, "pairs": [ { "id": "m11-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.0652 }, "b": { "src": "audio/m11-zh-CN-B.wav", "model": "s2-pro", "duration": 16.2046 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural and expressive delivery, capturing the warm and teasing tones better than Sample B, which sounds slightly more rigid.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery, capturing the warm and teasing tones better than Sample B, which sounds slightly more rigid." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with appropriate warmth and teasing tones. Differences in naturalness and expressiveness are negligible." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff and lacks the dynamic emotional shifts requested.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m11-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-ja-JP-A.wav", "model": "s2-pro", "duration": 19.7572 }, "b": { "src": "audio/m11-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.5715 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A features much more natural pitch accents and prosody for Japanese. Sample B suffers from noticeable pitch accent errors that make it sound non-native or robotic.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A features much more natural pitch accents and prosody for Japanese. Sample B suffers from noticeable pitch accent errors that make it sound non-native or robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it maintains accurate pronunciation throughout. Sample A mispronounces a word near the end." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural intonation and pacing throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pitch accents on several words like 'yoyaku' and 'madogiwa'.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '残して' (nokoshite) as 'nokosute'.", "severity": "major", "timestamp_seconds": 17.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural intonation and pacing throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional pitch accents is based on standard Tokyo Japanese.", "Both samples exhibit slightly non-native or robotic prosody, making naturalness difficult to rate highly." ] } }, { "id": "m11-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-ko-KR-A.wav", "model": "s2-pro", "duration": 19.6643 }, "b": { "src": "audio/m11-ko-KR-B.wav", "model": "s2.1-pro", "duration": 17.8068 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are intelligible and accurate to the text but suffer from similarly robotic prosody and unnatural pacing. Neither clearly outperforms the other.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text but suffer from similarly robotic prosody and unnatural pacing. Neither clearly outperforms the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text but suffer from robotic prosody and fail to convey the requested emotional shifts. They are tied in overall quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel mechanical throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech rhythm is stiff and unnatural.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Does not reflect the requested warm or teasing emotional shifts.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The phrasing and cadence are unnatural and mechanical.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and pauses sound unnatural and mechanical.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the warm, reassuring, and gently teasing tones requested.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit robotic prosody and unnatural phrasing.", "Both samples exhibit similar robotic prosody and lack the requested emotional shifts." ] } }, { "id": "m11-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-es-ES-A.wav", "model": "s2-pro", "duration": 16.6922 }, "b": { "src": "audio/m11-es-ES-B.wav", "model": "s2.1-pro", "duration": 15.6705 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples feature a strong non-native accent. Sample B is preferred for slightly more natural phrasing and less choppy prosody compared to Sample A.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples feature a strong non-native accent. Sample B is preferred for slightly more natural phrasing and less choppy prosody compared to Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe English accent that mispronounces Spanish phonemes. Sample A is slightly preferred for having marginally better flow and prosody than B's stilted delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'reservado' and 'galleta'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is somewhat choppy and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent applied to Spanish words, such as the 'r' in 'reservado' and 'rato'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat stilted and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, with English-like 'r' sounds.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent applied to Spanish words, such as the 'r' in 'reservado' and 'rato'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a strong non-native (American) accent, making it difficult to assess native es-ES pronunciation accurately.", "Both samples exhibit a strong non-native (English) accent, making it difficult to assess native Spanish prosody." ] } }, { "id": "m11-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-fr-FR-A.wav", "model": "s2-pro", "duration": 16.2975 }, "b": { "src": "audio/m11-fr-FR-B.wav", "model": "s2.1-pro", "duration": 15.0436 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a much more natural and accurate French pronunciation. Sample A suffers from a heavy foreign accent that distracts from the intended delivery.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a much more natural and accurate French pronunciation. Sample A suffers from a heavy foreign accent that distracts from the intended delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A has slightly better flow and less severe pronunciation issues than Sample B. Sample B sounds more robotic and has a stronger foreign accent." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign (English-like) accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Very strong non-native accent, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is disjointed and lacks natural flow.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'réservé' and 'pour'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional French accents is limited to general native-like expectations.", "Both samples exhibit strong non-native accents." ] } }, { "id": "m11-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.0637 }, "b": { "src": "audio/m11-de-DE-B.wav", "model": "s2-pro", "duration": 18.2479 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe English accent when speaking German, making them sound unnatural. Neither successfully captures the requested emotional delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent when speaking German, making them sound unnatural. Neither successfully captures the requested emotional delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a very strong foreign accent that impacts the naturalness of the German pronunciation. However, both are fully intelligible and preserve the target text accurately, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir Zeit' and 'reserviert'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir Zeit' and 'reserviert'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 16.0 } ] }, "limitations": [ "Both samples feature a very strong English accent that severely impacts the naturalness of the German pronunciation.", "Both samples feature a strong foreign accent, which affects naturalness but not strict intelligibility." ] } }, { "id": "m11-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.2263 }, "b": { "src": "audio/m11-pt-BR-B.wav", "model": "s2-pro", "duration": 19.0142 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples successfully deliver the text with clear audio and good intelligibility. However, both suffer from a non-native accent and lack the requested emotional shifts, making them equally usable but limited.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the text with clear audio and good intelligibility. However, both suffer from a non-native accent and lack the requested emotional shifts, making them equally usable but limited." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy foreign accent that affects pronunciation and naturalness. Neither successfully captures a native Brazilian Portuguese delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is relatively flat and does not clearly convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "The accent sounds somewhat non-native for Brazilian Portuguese.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent on Portuguese words, such as 'tranquilo' and 'lugares'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Cadence is somewhat stiff and unnatural.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speech remains mostly neutral, missing the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "The pronunciation has a noticeable non-native quality.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, particularly on 'Reservei' and 'lugares'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent for Brazilian Portuguese.", "Both samples exhibit a strong non-native (English) accent, making it difficult to assess native-level prosody." ] } }, { "id": "m11-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m11-ar-A.wav", "model": "s2.1-pro", "duration": 18.573 }, "b": { "src": "audio/m11-ar-B.wav", "model": "s2-pro", "duration": 19.6179 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has slightly better pronunciation and naturalness compared to Sample B, which struggles with the initial vowel sounds. Both exhibit some robotic prosody.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has slightly better pronunciation and naturalness compared to Sample B, which struggles with the initial vowel sounds. Both exhibit some robotic prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B features significantly better Arabic pronunciation and naturalness. Sample A suffers from a severe non-native accent that heavily degrades intelligibility." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses and cadence throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Minor vowel distortions in some words.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Reads 7:45 as 'سبعة خمسة وأربعين'.", "severity": "minor", "timestamp_seconds": 3.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Initial 'لا' pronounced with an incorrect vowel sound.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural phrasing and cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe non-native accent affects almost all words, e.g., 'لا داعي' and 'للاستعجال'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Reads 7:45 as 'سبعة خمسة وأربعين'.", "severity": "minor", "timestamp_seconds": 4.0 } ] }, "limitations": [ "Non-native speaker evaluating Arabic pronunciation nuances.", "I am evaluating the Arabic pronunciation and prosody based on standard conventions." ] } } ] }, { "id": "m12", "name": "Horgan", "slug": "horgan-267181", "gender": "Male", "rank": 12, "portrait": "portraits/m12.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 8 }, "pairs": [ { "id": "m12-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-zh-CN-A.wav", "model": "s2-pro", "duration": 15.6937 }, "b": { "src": "audio/m12-zh-CN-B.wav", "model": "s2.1-pro", "duration": 15.3919 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are highly intelligible and accurate to the text. However, both struggle to fully convey the requested emotional shifts, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. However, both struggle to fully convey the requested emotional shifts, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both lack the requested emotional shifts and sound somewhat robotic. There is no clear winner." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery remains relatively flat and does not clearly shift to a gently teasing tone at the end.", "severity": "minor", "timestamp_seconds": 11.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation sound slightly synthetic.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Does not effectively capture the warm or teasing tones requested.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Similar to A, the voice lacks the warmth and teasing qualities requested in the prompt.", "severity": "minor", "timestamp_seconds": 11.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks natural cadence.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m12-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-ja-JP-A.wav", "model": "s2.1-pro", "duration": 16.878 }, "b": { "src": "audio/m12-ja-JP-B.wav", "model": "s2-pro", "duration": 18.1318 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B correctly pronounces the target text. Sample A mispronounces 'ashita' (tomorrow) as 'asa' (morning), changing the meaning.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B correctly pronounces the target text. Sample A mispronounces 'ashita' (tomorrow) as 'asa' (morning), changing the meaning." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A reads the text accurately. Sample B mispronounces '明日' (ashita) as 'asa', which changes the meaning. Both samples exhibit slightly robotic prosody." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日' (ashita) as 'asa'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日' as 'asa' instead of 'ashita'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat unnatural.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly disjointed cadence throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, which affects naturalness but is evaluated primarily on intelligibility and text fidelity.", "I am not a native Japanese speaker, but the pronunciation error in Sample B is clear." ] } }, { "id": "m12-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-ko-KR-A.wav", "model": "s2-pro", "duration": 16.8315 }, "b": { "src": "audio/m12-ko-KR-B.wav", "model": "s2.1-pro", "duration": 16.6457 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly intelligible and accurate to the text, but both suffer from slightly robotic and stiff prosody. Neither successfully captures the nuanced emotional shifts requested in the prompt.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both suffer from slightly robotic and stiff prosody. Neither successfully captures the nuanced emotional shifts requested in the prompt." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A exhibits slightly more natural phrasing and pacing. Sample B sounds a bit more rigid in its delivery, though both are highly intelligible and accurate." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly mechanical throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The shift to 'gently teasing' is not clearly audible.", "severity": "minor", "timestamp_seconds": 11.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat rigid and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": 8.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech rhythm is somewhat unnatural and rigid.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the 'gently teasing' tone in the final sentences.", "severity": "minor", "timestamp_seconds": 11.8 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing in the middle sentences.", "severity": "minor", "timestamp_seconds": 6.0 } ] }, "limitations": [ "Both samples exhibit slightly robotic prosody, making it difficult to distinguish a clear winner." ] } }, { "id": "m12-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-es-ES-A.wav", "model": "s2.1-pro", "duration": 15.0204 }, "b": { "src": "audio/m12-es-ES-B.wav", "model": "s2-pro", "duration": 16.46 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a noticeable non-native accent but remain intelligible and follow the target text accurately. Neither clearly outperforms the other.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a noticeable non-native accent but remain intelligible and follow the target text accurately. Neither clearly outperforms the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe American English accent when reading the Spanish text, failing to sound like native Spanish (Spain) speakers. Neither successfully captures the requested emotional delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy American English accent throughout, e.g., 'prisa', 'reservado'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy American English accent throughout, e.g., 'prisa', 'reservado'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, making it difficult to assess subtle regional nuances." ] } }, { "id": "m12-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-fr-FR-A.wav", "model": "s2-pro", "duration": 14.556 }, "b": { "src": "audio/m12-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.0683 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a very strong American English accent when speaking French, severely impacting naturalness. Neither successfully captures the requested warm or teasing delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a very strong American English accent when speaking French, severely impacting naturalness. Neither successfully captures the requested warm or teasing delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent and robotic prosody. Neither successfully conveys the requested warm or teasing delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong American accent throughout, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is flat and mechanical, lacking the requested warmth.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong American accent throughout, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is flat and mechanical, lacking the requested warmth.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a strong non-native accent.", "Both samples exhibit a severe foreign (American) accent, making it difficult to assess native French prosody." ] } }, { "id": "m12-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-de-DE-A.wav", "model": "s2-pro", "duration": 16.9244 }, "b": { "src": "audio/m12-de-DE-B.wav", "model": "s2.1-pro", "duration": 15.508 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a strong foreign accent and mispronounce 'Keks' as the English word 'cakes'. Sample A is preferred because Sample B adds an incorrect suffix to the number, saying 'fünfundvierzigst'.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples feature a strong foreign accent and mispronounce 'Keks' as the English word 'cakes'. Sample A is preferred because Sample B adds an incorrect suffix to the number, saying 'fünfundvierzigst'." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a very heavy English accent that severely impacts the pronunciation of the German text. Neither sample provides a convincing or native-sounding delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Lass' as 'Lest'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 14.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir' sounds like 'Last year'.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'fünfundvierzig' as 'fünfundvierzigst'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 13.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir' sounds like 'Last year'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a very heavy foreign accent that affects intelligibility." ] } }, { "id": "m12-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.412 }, "b": { "src": "audio/m12-pt-BR-B.wav", "model": "s2-pro", "duration": 17.8068 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are intelligible and accurate to the text, but both suffer from a noticeable non-native accent and flat delivery that fails to capture the requested emotional shifts.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and accurate to the text, but both suffer from a noticeable non-native accent and flat delivery that fails to capture the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples exhibit a heavy non-native accent that impacts pronunciation. Sample A is preferred because it has slightly better flow and less robotic phrasing than Sample B." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and does not convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'pressa' and 'Reservei'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is choppy and stilted throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery lacks the requested emotional shifts and sounds somewhat mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'pressa' and 'Reservei' (using an English 'r' sound).", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, which affects naturalness.", "Both samples feature a strong non-native (likely English) accent, which affects the naturalness of the Brazilian Portuguese pronunciation." ] } }, { "id": "m12-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m12-ar-A.wav", "model": "s2-pro", "duration": 17.621 }, "b": { "src": "audio/m12-ar-B.wav", "model": "s2.1-pro", "duration": 17.8764 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred because it correctly conjugates the verb 'حجزت' (I booked), whereas Sample B incorrectly pronounces it as 'she booked'. Both suffer from unnatural pauses.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because it correctly conjugates the verb 'حجزت' (I booked), whereas Sample B incorrectly pronounces it as 'she booked'. Both suffer from unnatural pauses." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit noticeable synthetic prosody and slightly robotic pacing. They both convey the text adequately but lack the requested warmth and teasing nuances." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is somewhat stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Does not effectively capture the requested warm or gently teasing tones.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as 'hajazat' (she booked) instead of 'hajaztu' (I booked).", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing and pauses feel slightly unnatural and mechanical.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit unnatural pauses and robotic prosody.", "Evaluation of Arabic number normalization and exact case endings may vary by regional standards." ] } } ] }, { "id": "m13", "name": "陳灝", "slug": "love-ab00ae", "gender": "Male", "rank": 13, "portrait": "portraits/m13.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 8 }, "pairs": [ { "id": "m13-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-zh-CN-A.wav", "model": "s2-pro", "duration": 17.7603 }, "b": { "src": "audio/m13-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.46 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples provide clear and accurate readings of the text. Sample B is preferred for its slightly more natural pacing and a marginally better attempt at the requested emotional shift.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Both samples provide clear and accurate readings of the text. Sample B is preferred for its slightly more natural pacing and a marginally better attempt at the requested emotional shift." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly with good intelligibility and text fidelity. However, both lack the distinct shift in expressiveness requested and sound slightly robotic, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is very subtle.", "severity": "minor", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence feels slightly unnatural and mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the gentle teasing tone at the end.", "severity": "minor", "timestamp_seconds": 13.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The teasing tone is present but could be more pronounced.", "severity": "minor", "timestamp_seconds": 12.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat rigid throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not clearly audible.", "severity": "minor", "timestamp_seconds": 12.0 } ] }, "limitations": [ "Subtle emotional shifts are difficult to evaluate objectively." ] } }, { "id": "m13-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-ja-JP-A.wav", "model": "s2-pro", "duration": 18.991 }, "b": { "src": "audio/m13-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.8037 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from severe and frequent pronunciation errors on basic Japanese vocabulary, rendering them equally unusable.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from severe and frequent pronunciation errors on basic Japanese vocabulary, rendering them equally unusable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples fail completely at basic Japanese pronunciation and kanji reading, resulting in numerous non-words and severe text fidelity issues. Neither is usable." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 2, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "2.0/5 mean text-fidelity score", "B": "2.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '急が' as 'shiga', '明日' as 'ishido', and '45分' as 'yongo hin'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '二人で' as 'urari de'.", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' as 'shikanakute' and '明日' as 'ishiro'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '45分' as 'yongohin'.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '二人で' as 'uraride'.", "severity": "major", "timestamp_seconds": 11.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '急が' as 'isanga', '明日' as 'minan', and '45分' as 'yonshugo han'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '二人で' as 'ari de'.", "severity": "major", "timestamp_seconds": 11.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' as 'isaganakute' and '明日' as 'minan'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '45分' as 'yonshugohan'.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '二人で' as 'haride'.", "severity": "major", "timestamp_seconds": 11.0 } ] }, "limitations": [ "Both models struggle significantly with basic Japanese kanji readings.", "Both models exhibit severe kanji reading and pronunciation errors, making them unusable for Japanese." ] } }, { "id": "m13-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-ko-KR-A.wav", "model": "s2-pro", "duration": 19.9662 }, "b": { "src": "audio/m13-ko-KR-B.wav", "model": "s2.1-pro", "duration": 18.5034 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural phrasing and pacing compared to Sample A. Both samples are highly intelligible and follow the text accurately.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly more natural phrasing and pacing compared to Sample A. Both samples are highly intelligible and follow the text accurately." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A delivers a more natural and expressive performance that better aligns with the requested warm and teasing tones. Sample B sounds slightly more robotic and flat." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat flat and lack the requested warmth and teasing.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [ "I am not a native Korean speaker, so subtle accent nuances may be missed.", "I am not a native Korean speaker, but I can assess general prosody, intelligibility, and text fidelity." ] } }, { "id": "m13-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-es-ES-A.wav", "model": "s2.1-pro", "duration": 17.0637 }, "b": { "src": "audio/m13-es-ES-B.wav", "model": "s2-pro", "duration": 17.8068 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from robotic prosody and non-native pronunciation. Sample A is slightly preferred as its pronunciation is marginally closer to the target locale than Sample B's heavy accent.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from robotic prosody and non-native pronunciation. Sample A is slightly preferred as its pronunciation is marginally closer to the target locale than Sample B's heavy accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A has slightly better pacing and naturalness than Sample B, which sounds more robotic and flat. Both struggle with the requested emotional delivery and exhibit non-native pronunciation." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Non-native pronunciation of 'He reservado' and general accent.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted pacing and unnatural pauses throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is highly mechanical with unnatural cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The tone is flat and ignores the performance directions.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects the natural flow of the Spanish text.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent, e.g., 'He' pronounced like 'Hey'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy and unnatural delivery.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and pauses feel somewhat unnatural and mechanical.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and gently teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit strong non-native accents and robotic prosody.", "Both samples exhibit a noticeable non-native accent and robotic prosody." ] } }, { "id": "m13-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-fr-FR-A.wav", "model": "s2-pro", "duration": 16.7154 }, "b": { "src": "audio/m13-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.9042 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody and unnatural pauses between phrases. Neither captures the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and unnatural pauses between phrases. Neither captures the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better flow and naturalness. Sample B suffers from robotic pacing and unnatural pauses between phrases." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and choppy delivery throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural, rigid pauses between phrases.", "severity": "major", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Non-native accent affects overall rhythm.", "severity": "minor", "timestamp_seconds": 1.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy phrasing with unnatural gaps between sentences.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and ignores the emotional cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, particularly on nasal vowels.", "severity": "minor", "timestamp_seconds": 0.5 } ] }, "limitations": [ "Both samples exhibit highly unnatural, robotic prosody with excessive pausing.", "Both samples exhibit non-native accents, making it difficult to assess perfect French prosody." ] } }, { "id": "m13-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.4352 }, "b": { "src": "audio/m13-de-DE-B.wav", "model": "s2-pro", "duration": 18.8052 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples are highly intelligible and accurate to the text, but both suffer from a somewhat robotic delivery that fails to capture the requested emotional shifts.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but both suffer from a somewhat robotic delivery that fails to capture the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. However, both sound somewhat robotic and fail to fully capture the requested emotional shift." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel somewhat mechanical, lacking natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm, reassuring tone or the shift to gentle teasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested shift to gentle teasing.", "severity": "minor", "timestamp_seconds": 13.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech has a slightly synthetic cadence with unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The tone remains relatively flat and does not reflect the requested emotional cues.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not clearly audible.", "severity": "minor", "timestamp_seconds": 14.0 } ] }, "limitations": [ "Both samples exhibit limited expressiveness and do not clearly convey the requested shift to gentle teasing." ] } }, { "id": "m13-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-pt-BR-A.wav", "model": "s2-pro", "duration": 19.2232 }, "b": { "src": "audio/m13-pt-BR-B.wav", "model": "s2.1-pro", "duration": 18.2712 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred for its slightly more natural pacing and expressiveness. Sample B sounds more robotic and flat, failing to capture the requested warm and teasing tones.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred for its slightly more natural pacing and expressiveness. Sample B sounds more robotic and flat, failing to capture the requested warm and teasing tones." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese. Sample A is preferred because it has slightly better naturalness and phrasing compared to the more robotic Sample B." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery lacks the requested warmth and teasing tone, sounding somewhat flat.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "The accent is European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Uses European Portuguese pronunciation instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is stiff and lacks the requested warm and teasing delivery.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is stiff and unnatural, lacking human-like cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm and teasing directions, sounding very monotone.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "Uses a European Portuguese accent rather than the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Uses European Portuguese pronunciation instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale.", "Both samples use a European Portuguese accent instead of the requested Brazilian Portuguese locale." ] } }, { "id": "m13-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m13-ar-A.wav", "model": "s2.1-pro", "duration": 21.104 }, "b": { "src": "audio/m13-ar-B.wav", "model": "s2-pro", "duration": 20.4306 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred because Sample B cuts off the end of the first sentence. Both samples suffer from unnatural pauses and mispronunciations of certain words.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because Sample B cuts off the end of the first sentence. Both samples suffer from unnatural pauses and mispronunciations of certain words." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it correctly pronounces the first sentence, whereas Sample A has a clear pronunciation error on 'للاستعجال'. Both samples suffer from robotic prosody and unnatural pauses." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'مقعدين' (maq'adayn) as 'muqa'adin'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 8.0 } ], "B": [ { "order": "AB", "category": "cutoff", "evidence": "The word 'للاستعجال' is cut off at the end.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'مقعدين' (maq'adayn) as 'muqa'adin'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'للاستعجال' as something like 'للاستيعاه'.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ] }, "limitations": [ "I am evaluating the Arabic pronunciation and prosody based on standard rules, but regional variations may exist.", "I am not a native speaker of Arabic, but can identify clear pronunciation errors and prosodic unnaturalness." ] } } ] }, { "id": "m14", "name": "久世 綾人", "slug": "kuze-ayato-71243c48-5e04-451f-9421-0730977af1a8", "gender": "Male", "rank": 14, "portrait": "portraits/m14.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 6 }, "pairs": [ { "id": "m14-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-zh-CN-A.wav", "model": "s2-pro", "duration": 12.4197 }, "b": { "src": "audio/m14-zh-CN-B.wav", "model": "s2.1-pro", "duration": 11.8625 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly intelligible and accurate to the text. However, both fail to convey the requested emotional shifts, sounding relatively flat and slightly robotic.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. However, both fail to convey the requested emotional shifts, sounding relatively flat and slightly robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples fail to convey the requested emotional cues, sounding relatively flat. Sample B is slightly preferred for its marginally better pacing and naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery remains neutral throughout, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery remains mostly neutral, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to capture the warm and teasing tones requested in the prompt.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Prosody is somewhat flat and unnatural.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and rushed, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples lack the requested emotional shifts." ] } }, { "id": "m14-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-ja-JP-A.wav", "model": "s2.1-pro", "duration": 20.152 }, "b": { "src": "audio/m14-ja-JP-B.wav", "model": "s2-pro", "duration": 17.1102 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B follows the text accurately, while Sample A adds an extra word at the beginning. Both provide a suitable delivery.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B follows the text accurately, while Sample A adds an extra word at the beginning. Both provide a suitable delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A follows the text perfectly and delivers a natural, warm performance. Sample B adds an unscripted word at the beginning and sounds slightly overly breathy." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4.5, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4.5, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_addition", "evidence": "Added 'はい' (Hai) at the beginning of the recording.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "text_addition", "evidence": "Added 'はい' (Hai) at the beginning before '急がなくていいよ'.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [] }, "limitations": [] } }, { "id": "m14-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-ko-KR-A.wav", "model": "s2.1-pro", "duration": 17.8996 }, "b": { "src": "audio/m14-ko-KR-B.wav", "model": "s2-pro", "duration": 17.7835 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a slightly more natural and expressive delivery, capturing the intended emotional shifts better than Sample A, which sounds a bit flatter.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers a slightly more natural and expressive delivery, capturing the intended emotional shifts better than Sample A, which sounds a bit flatter." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery, better capturing the requested warm and teasing tones compared to Sample B." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones, sounding somewhat flat.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [] } }, { "id": "m14-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-es-ES-A.wav", "model": "s2-pro", "duration": 14.1148 }, "b": { "src": "audio/m14-es-ES-B.wav", "model": "s2.1-pro", "duration": 14.6953 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer pronunciation and better audio quality. Sample A has a noticeable pronunciation error on 'reservado' and sounds more synthetic.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides clearer pronunciation and better audio quality. Sample A has a noticeable pronunciation error on 'reservado' and sounds more synthetic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is more intelligible and has better audio quality. Sample B suffers from a stronger foreign accent, choppy prosody, and noticeable background hiss." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronunciation of 'reservado', sounds distorted.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with distorted vowels, e.g., 'reservado'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Very choppy and unnatural phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and synthetic artifacts.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pauses and intonation patterns throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects the naturalness of the Spanish.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, making it difficult to assess native es-ES prosody." ] } }, { "id": "m14-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.6473 }, "b": { "src": "audio/m14-fr-FR-B.wav", "model": "s2-pro", "duration": 14.4631 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a strong non-native accent and an overly breathy delivery. Sample B is slightly preferred due to marginally better intelligibility and pronunciation of certain words like 'tranquille'.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a strong non-native accent and an overly breathy delivery. Sample B is slightly preferred due to marginally better intelligibility and pronunciation of certain words like 'tranquille'." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe non-native accent that drastically reduces intelligibility and naturalness for the requested French locale. Neither is usable." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects words like 'tranquille' (sounds like 'tankil').", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is overly breathy and whispered rather than naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe non-native accent throughout, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent impacts the natural flow of the French text.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Speech is somewhat whispered, detracting from a natural conversational tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe non-native accent throughout, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent that impacts the evaluation of French (France) pronunciation.", "Both samples feature a severe non-native accent that heavily impacts intelligibility." ] } }, { "id": "m14-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-de-DE-A.wav", "model": "s2-pro", "duration": 18.1086 }, "b": { "src": "audio/m14-de-DE-B.wav", "model": "s2.1-pro", "duration": 15.9027 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a much more natural and expressive delivery that aligns with the requested tone, despite some breathy artifacts. Sample B is highly robotic, choppy, and fails to convey the intended emotion.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a much more natural and expressive delivery that aligns with the requested tone, despite some breathy artifacts. Sample B is highly robotic, choppy, and fails to convey the intended emotion." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better pacing and naturalness. Sample B suffers from overly long pauses and an exaggerated, dramatic whisper that detracts from the intended delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The word 'reden' sounds slightly distorted or mispronounced.", "severity": "minor", "timestamp_seconds": 9.5 }, { "order": "AB", "category": "other", "evidence": "The voice has a slightly exaggerated breathy quality.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnaturally long pauses between phrases.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly whispered and dramatic, missing the gentle teasing tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is very choppy and synthetic throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm or teasing tones requested, sounding completely flat.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat choppy and the delivery feels slightly mechanical.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a highly breathy, almost whispered vocal style." ] } }, { "id": "m14-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-pt-BR-A.wav", "model": "s2-pro", "duration": 17.7603 }, "b": { "src": "audio/m14-pt-BR-B.wav", "model": "s2.1-pro", "duration": 17.3191 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody, unnatural pauses, and a strong non-native accent that distorts pronunciation. Neither successfully captures the requested warm or teasing delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody, unnatural pauses, and a strong non-native accent that distorts pronunciation. Neither successfully captures the requested warm or teasing delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A has better pacing and flow than Sample B, which suffers from excessive pausing. Both struggle with the pronunciation of 'Reservei'." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Highly unnatural, choppy phrasing throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent distorts words like 'Reservei'.", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "BA", "category": "pronunciation", "evidence": "'Reservei' is mispronounced.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Excessive pausing and slow pacing.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy, disjointed delivery with unnatural pauses.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent affects the pronunciation of 'Reservei' and other words.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Reservei' is mispronounced.", "severity": "major", "timestamp_seconds": 2.0 } ] }, "limitations": [ "Both samples feature a strong non-native accent that affects pronunciation.", "Both samples struggle with the pronunciation of 'Reservei'." ] } }, { "id": "m14-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m14-ar-A.wav", "model": "s2-pro", "duration": 20.2913 }, "b": { "src": "audio/m14-ar-B.wav", "model": "s2.1-pro", "duration": 19.0838 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe pronunciation issues and a heavy foreign accent that makes the Arabic largely unintelligible. Sample B omits the minutes in the time, while Sample A struggles heavily with the numbers, resulting in a tie for poor quality.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe pronunciation issues and a heavy foreign accent that makes the Arabic largely unintelligible. Sample B omits the minutes in the time, while Sample A struggles heavily with the numbers, resulting in a tie for poor quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples fail to correctly pronounce the time '7:45'. Sample A is preferred for having slightly more natural prosody, whereas Sample B is highly disjointed and reads the time as individual digits." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "2.0/5 mean text-fidelity score", "B": "2.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent distorts most Arabic phonemes, making words like 'للاستعجال' and 'النافذة' hard to recognize.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_substitution", "evidence": "The time '7:45' is read with distorted and incorrect numbers.", "severity": "major", "timestamp_seconds": 3.5 }, { "order": "BA", "category": "text_substitution", "evidence": "Expected '7:45', heard 'saba'a arba'a khamsa' (7 4 5).", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Speech is highly disjointed with unnatural pauses between words.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent distorts phonemes throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_omission", "evidence": "Omits the minutes '45' when reading the time '7:45'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Expected '7:45', heard an unintelligible phrase ending in 'sab'a'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'هادئة' (hadi'a).", "severity": "minor", "timestamp_seconds": 11.5 } ] }, "limitations": [ "Both samples exhibit a severe non-native accent that heavily distorts Arabic phonemes.", "Both models struggle significantly with Arabic numerals and time formatting." ] } } ] }, { "id": "m15", "name": "Rhett Callahan", "slug": "rhett-callahan-7c6b84", "gender": "Male", "rank": 15, "portrait": "portraits/m15.webp", "language": "8 languages", "popularity": { "callers": 6, "calls": 6 }, "pairs": [ { "id": "m15-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.8547 }, "b": { "src": "audio/m15-zh-CN-B.wav", "model": "s2-pro", "duration": 16.7619 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe foreign accent and incorrect tones, making them equally unnatural for the target locale. There is no clear winner.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent and incorrect tones, making them equally unnatural for the target locale. There is no clear winner." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from severe foreign accents and widespread tone errors, making them difficult to understand. Sample A is slightly preferred for having marginally better prosody and expressiveness compared to the choppier delivery of Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 2, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent and incorrect tones throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent and incorrect tones throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural cadence.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent and incorrect tones throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent and incorrect tones throughout the recording.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a severe foreign accent.", "Evaluation of subtle tone nuances may be limited, though the severe foreign accent in both samples is clearly identifiable." ] } }, { "id": "m15-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-ja-JP-A.wav", "model": "s2.1-pro", "duration": 21.9631 }, "b": { "src": "audio/m15-ja-JP-B.wav", "model": "s2-pro", "duration": 21.0111 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples have a heavy foreign accent that detracts from naturalness. Sample B is preferred because it includes a natural-sounding laugh that effectively captures the 'gently teasing' direction.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a heavy foreign accent that detracts from naturalness. Sample B is preferred because it includes a natural-sounding laugh that effectively captures the 'gently teasing' direction." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred for better pronunciation and expressiveness. Sample B mispronounces the first word, while Sample A includes a laugh that fits the teasing direction well." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, affecting pitch accent and vowel lengths.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'izaganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent throughout, affecting pitch accent and vowel lengths.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, though words remain intelligible.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong foreign accent, making the Japanese sound unnatural, though the words remain mostly intelligible.", "Both samples feature a heavy non-native accent, which affects naturalness but is evaluated based on intelligibility and text fidelity." ] } }, { "id": "m15-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-ko-KR-A.wav", "model": "s2.1-pro", "duration": 20.5003 }, "b": { "src": "audio/m15-ko-KR-B.wav", "model": "s2-pro", "duration": 20.7789 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have a severe foreign accent. Sample A is preferred because Sample B includes an unnatural, jarring whisper for the teasing section that does not fit the intended delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples have a severe foreign accent. Sample A is preferred because Sample B includes an unnatural, jarring whisper for the teasing section that does not fit the intended delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent that heavily impacts naturalness and pronunciation. Neither is suitable for a native Korean use case, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The entire sample is spoken with a heavy non-native accent, affecting vowel and consonant clarity.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is relatively flat and does not clearly convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "The entire sample is spoken with a heavy non-native accent.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The shift to a whisper at '대신 약속 하나만 해 줘' sounds creepy rather than gently teasing.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong non-native accent, which significantly impacts naturalness.", "Both samples exhibit a severe non-native (English) accent, making it difficult to evaluate native prosody." ] } }, { "id": "m15-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-es-ES-A.wav", "model": "s2.1-pro", "duration": 18.6659 }, "b": { "src": "audio/m15-es-ES-B.wav", "model": "s2-pro", "duration": 16.4368 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe foreign accent that impacts naturalness. Sample A is preferred as its pacing and tone better match the 'warm, reassuring' direction, whereas Sample B sounds slightly rushed.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a severe foreign accent that impacts naturalness. Sample A is preferred as its pacing and tone better match the 'warm, reassuring' direction, whereas Sample B sounds slightly rushed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent that heavily impacts naturalness and pronunciation. Neither successfully captures the requested es-ES locale or the intended emotional delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American English accent throughout, e.g., 'galleta' pronounced with an English 'l' and 'y' sound.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, mispronouncing vowels and consonants like 'll' in 'galleta'.", "severity": "major", "timestamp_seconds": 16.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing emotional shifts, sounding robotic.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat rushed and lacks the requested 'warm, reassuring' tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'galleta' pronounced with an English 'l' and 'y' sound.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and does not convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a severe American English accent when speaking Spanish.", "Both samples feature a very strong non-native (English) accent, making it difficult to assess native es-ES prosody." ] } }, { "id": "m15-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-fr-FR-A.wav", "model": "s2.1-pro", "duration": 17.3424 }, "b": { "src": "audio/m15-fr-FR-B.wav", "model": "s2-pro", "duration": 17.3888 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe non-native accent that degrades naturalness. Sample A is preferred for having slightly smoother prosody and better flow than the choppier Sample B.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a severe non-native accent that degrades naturalness. Sample A is preferred for having slightly smoother prosody and better flow than the choppier Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe foreign accent that distorts French pronunciation. Sample A is slightly preferred for having marginally better prosody and expressiveness compared to B's flatter delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Very strong non-native accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout, making many words difficult to understand.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is relatively flat and does not convey the requested warmth or teasing.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy and unnatural phrasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent throughout, distorting words like 'places' and 'fenêtre'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature an extremely heavy non-native (American) accent, making it difficult to assess native French prosody.", "Both samples feature a severe non-native accent that heavily impacts intelligibility and naturalness." ] } }, { "id": "m15-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-de-DE-A.wav", "model": "s2-pro", "duration": 17.5281 }, "b": { "src": "audio/m15-de-DE-B.wav", "model": "s2.1-pro", "duration": 17.5978 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from the exact same severe pronunciation error at the beginning, reading 'Lass dir Zeit' as if it were English ('Last year sight'). Both also have a heavy foreign accent.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from the exact same severe pronunciation error at the beginning, reading 'Lass dir Zeit' as if it were English ('Last year sight'). Both also have a heavy foreign accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples exhibit a very strong English accent that severely impacts intelligibility and naturalness in German. Neither is clearly better." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Keks' pronounced like 'cakes', 'genießen' mispronounced.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Keks' pronounced like 'cakes', 'zwei' like 'schwai'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both models exhibit a very strong American accent when speaking German.", "Both samples feature an extremely heavy foreign accent, making it difficult to assess native-level prosody." ] } }, { "id": "m15-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-pt-BR-A.wav", "model": "s2-pro", "duration": 18.8052 }, "b": { "src": "audio/m15-pt-BR-B.wav", "model": "s2.1-pro", "duration": 19.2928 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe foreign accent and robotic prosody that detracts from the intended delivery. Neither successfully conveys the requested emotions.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent and robotic prosody that detracts from the intended delivery. Neither successfully conveys the requested emotions." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples feature a heavy non-native accent that impacts naturalness. Sample B is preferred for having slightly better flow and a softer tone that better approximates the requested warm delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth and teasing tone.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_omission", "evidence": "Omitted 'às' before the time.", "severity": "minor", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is loud and flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong foreign accent, making nuanced evaluation of native prosody difficult.", "Both samples exhibit a strong non-native accent when speaking Portuguese." ] } }, { "id": "m15-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m15-ar-A.wav", "model": "s2-pro", "duration": 21.9399 }, "b": { "src": "audio/m15-ar-B.wav", "model": "s2.1-pro", "duration": 21.2201 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples struggle significantly with Modern Standard Arabic pronunciation, substituting phonemes with colloquial or non-native variants. Neither successfully captures the requested emotional shifts.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples struggle significantly with Modern Standard Arabic pronunciation, substituting phonemes with colloquial or non-native variants. Neither successfully captures the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and heavy non-native accents. Sample A is preferred as it has slightly better audio quality and is less distorted than Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2, "audioQuality": 2.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Substitutes 'j' with 'y' (e.g., للاستعجال, حجزت) and 'q' with 'g' (مقعدين), which is inappropriate for MSA.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or gently teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "The audio is loud and distorted, especially at the beginning.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly unnatural, robotic cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent makes some words difficult to understand.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces several words, such as 'مقعدين' with a 'k' sound instead of 'q'.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is unnatural and choppy throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is highly robotic with unnatural pauses, such as between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit strong non-standard or colloquial accents that affect MSA phoneme realization.", "Both samples feature a heavy non-native accent that affects naturalness." ] } } ] }, { "id": "m16", "name": "Dean Winchester", "slug": "dean-winchester-509185", "gender": "Male", "rank": 16, "portrait": "portraits/m16.webp", "language": "8 languages", "popularity": { "callers": 5, "calls": 9 }, "pairs": [ { "id": "m16-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-zh-CN-A.wav", "model": "s2.1-pro", "duration": 15.2061 }, "b": { "src": "audio/m16-zh-CN-B.wav", "model": "s2-pro", "duration": 14.0916 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from a severe non-native accent and incorrect tones throughout. Sample B is slightly preferred for having marginally better pacing and a bit more expression than A's completely flat delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe non-native accent and incorrect tones throughout. Sample B is slightly preferred for having marginally better pacing and a bit more expression than A's completely flat delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a much more natural and expressive delivery that aligns well with the requested emotional shifts. Sample B sounds flat, robotic, and lacks the appropriate warmth and teasing tones." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent with incorrect tones on most words, such as '慢慢来' and '不着急'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat, robotic, and lacks natural phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall delivery is flat and lacks natural cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing emotions.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent with incorrect tones on most words, similar to A.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing emotions, though slightly more dynamic than A.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent with pervasive tone errors, making it difficult to assess native-level naturalness." ] } }, { "id": "m16-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-ja-JP-A.wav", "model": "s2.1-pro", "duration": 16.1814 }, "b": { "src": "audio/m16-ja-JP-B.wav", "model": "s2-pro", "duration": 16.5065 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more natural and expressive delivery that better captures the warm and teasing directions. Sample A sounds slightly stiffer in comparison.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides a more natural and expressive delivery that better captures the warm and teasing directions. Sample A sounds slightly stiffer in comparison." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a warmer, more natural delivery that better fits the requested tone. Sample B has noticeable background noise and slightly robotic pacing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is slightly stiff and lacks the full warmth requested.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background noise and slight distortion throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat rigid and lacks natural warmth.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [] } }, { "id": "m16-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-ko-KR-A.wav", "model": "s2-pro", "duration": 15.1597 }, "b": { "src": "audio/m16-ko-KR-B.wav", "model": "s2.1-pro", "duration": 13.3485 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers better audio clarity and more natural pacing. Sample B sounds slightly muffled and rushed in its delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers better audio clarity and more natural pacing. Sample B sounds slightly muffled and rushed in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers clearer audio quality and a more natural, expressive delivery that better captures the requested warmth and teasing tone. Sample A suffers from noticeable background noise and a flatter prosody." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "other", "evidence": "Slight room tone or reverb present, but not highly disruptive.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Slightly muffled audio quality throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background noise and slight distortion throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing and intonation feel somewhat flat and mechanical.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m16-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-es-ES-A.wav", "model": "s2-pro", "duration": 14.2077 }, "b": { "src": "audio/m16-es-ES-B.wav", "model": "s2.1-pro", "duration": 14.4631 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from a heavy foreign accent, failing to meet the locale expectation. Sample A is slightly preferred due to cleaner audio, whereas B has noticeable noise gating.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a heavy foreign accent, failing to meet the locale expectation. Sample A is slightly preferred due to cleaner audio, whereas B has noticeable noise gating." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred due to clearer audio quality. Both samples feature a noticeable non-native accent, but B is more intelligible and lacks the muffled quality of A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, particularly on 'r' sounds (e.g., 'prisa', 'reservado').", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, particularly on 'r' sounds.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable noise gating and background hum cutting in and out.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Audio sounds slightly muffled with a low-quality feel.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong English accent rather than the requested Spanish (Spain) locale.", "Both samples exhibit a strong non-native accent, making it difficult to assess native es-ES prosody." ] } }, { "id": "m16-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-fr-FR-A.wav", "model": "s2.1-pro", "duration": 12.8609 }, "b": { "src": "audio/m16-fr-FR-B.wav", "model": "s2-pro", "duration": 14.138 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from severe pronunciation issues, sounding like an English speaker reading French phonetically. Neither is acceptable for native French speech.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe pronunciation issues, sounding like an English speaker reading French phonetically. Neither is acceptable for native French speech." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy English accent and mispronounce the opening phrase 'Prends ton temps'. Neither successfully captures native French phonetics, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'réservé'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Prends ton temps' is pronounced with English phonetics.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'fenêtre' is pronounced with an added 'n' sound at the end.", "severity": "minor", "timestamp_seconds": 6.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Prends ton temps' is pronounced with English phonetics.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a very heavy non-native accent, making the French difficult to understand.", "Both samples exhibit a strong non-native (English) accent, making it difficult to assess native French prosody." ] } }, { "id": "m16-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-de-DE-A.wav", "model": "s2-pro", "duration": 13.7432 }, "b": { "src": "audio/m16-de-DE-B.wav", "model": "s2.1-pro", "duration": 14.1612 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from the exact same severe pronunciation error at the beginning, reading the German phrase as English words. They are otherwise comparable in quality.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from the exact same severe pronunciation error at the beginning, reading the German phrase as English words. They are otherwise comparable in quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from a heavy foreign accent that impacts naturalness. Sample B is preferred because it maintains better text fidelity, whereas Sample A substitutes a word." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, particularly noticeable on 'Lass dir Zeit'.", "severity": "major", "timestamp_seconds": 0.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Heard 'und versprich mir' instead of 'Aber versprich mir'.", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, especially on 'Lass dir Zeit'.", "severity": "major", "timestamp_seconds": 0.5 } ] }, "limitations": [ "Both samples exhibit a very strong foreign (English) accent, affecting naturalness and pronunciation." ] } }, { "id": "m16-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-pt-BR-A.wav", "model": "s2-pro", "duration": 14.765 }, "b": { "src": "audio/m16-pt-BR-B.wav", "model": "s2.1-pro", "duration": 14.7185 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly better audio quality and naturalness. Sample B suffers from more noticeable robotic artifacts and a muffled sound, making it less pleasant to listen to.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly better audio quality and naturalness. Sample B suffers from more noticeable robotic artifacts and a muffled sound, making it less pleasant to listen to." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples use a European Portuguese accent instead of the requested Brazilian Portuguese. Sample B is preferred due to clearer audio quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and minor robotic artifacts throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Muffled audio quality with noticeable robotic distortion.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and lacks the requested warmth and teasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Audio is muffled with noticeable background hiss.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m16-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m16-ar-A.wav", "model": "s2.1-pro", "duration": 17.5281 }, "b": { "src": "audio/m16-ar-B.wav", "model": "s2-pro", "duration": 17.5281 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers clearer audio quality and slightly better expressiveness. Both samples suffer from an unnatural pause mid-sentence, but B is overall more pleasant to listen to.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers clearer audio quality and slightly better expressiveness. Both samples suffer from an unnatural pause mid-sentence, but B is overall more pleasant to listen to." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples share identical pronunciation errors and unnatural pauses. Neither demonstrates a clear advantage in naturalness or expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 6.5 }, { "order": "AB", "category": "distortion", "evidence": "Slightly muffled audio with background hiss.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'للاستعجال' with an 'h' sound instead of 'ain'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'للاستعجال' with an 'h' sound instead of 'ain'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 } ] }, "limitations": [ "Non-native Arabic speaker; evaluated based on general phonetic and prosodic principles.", "Both samples exhibit similar mispronunciations and unnatural pauses, making differentiation difficult." ] } } ] }, { "id": "m17", "name": "Dante Moretti", "slug": "dante-moretti-97a7e1", "gender": "Male", "rank": 17, "portrait": "portraits/m17.webp", "language": "8 languages", "popularity": { "callers": 5, "calls": 7 }, "pairs": [ { "id": "m17-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.9012 }, "b": { "src": "audio/m17-zh-CN-B.wav", "model": "s2-pro", "duration": 15.6241 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a strong non-native accent with robotic prosody and incorrect tones. Neither captures the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent with robotic prosody and incorrect tones. Neither captures the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a severe non-native accent with incorrect tones and unnatural prosody for Mandarin. They are equally intelligible but lack native naturalness." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Flat intonation and incorrect tones throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing emotional shifts.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with frequent tone errors throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Flat intonation and incorrect tones throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing emotional shifts.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent with frequent tone errors throughout.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, making tone evaluation challenging.", "Both samples feature a very strong non-native accent with frequent tone errors." ] } }, { "id": "m17-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-ja-JP-A.wav", "model": "s2-pro", "duration": 20.1752 }, "b": { "src": "audio/m17-ja-JP-B.wav", "model": "s2.1-pro", "duration": 18.7355 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A has significantly better Japanese pronunciation and pitch accent. Sample B suffers from a strong foreign accent and incorrect intonation throughout, making it sound highly unnatural.", "confidence": 0.9, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A has significantly better Japanese pronunciation and pitch accent. Sample B suffers from a strong foreign accent and incorrect intonation throughout, making it sound highly unnatural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a heavy foreign accent and robotic prosody. Neither captures the requested emotional delivery or natural Japanese phonetics, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly flat intonation and minor unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout the entire recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Flat, unnatural phrasing that lacks the requested warmth and teasing.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent with incorrect pitch accents on many words (e.g., 'yoyaku', 'madogiwa').", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Stilted and unnatural cadence throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy foreign accent throughout the entire recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Flat, unnatural phrasing that lacks the requested warmth and teasing.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a severe foreign accent, making it difficult to assess native-level prosody." ] } }, { "id": "m17-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-ko-KR-A.wav", "model": "s2.1-pro", "duration": 18.1086 }, "b": { "src": "audio/m17-ko-KR-B.wav", "model": "s2-pro", "duration": 17.7835 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are highly intelligible and accurate to the text. They both struggle slightly with natural pacing and the requested emotional shifts, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text. They both struggle slightly with natural pacing and the requested emotional shifts, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and expressiveness, capturing the intended tone more effectively than Sample A." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is somewhat rigid and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not distinctly audible.", "severity": "minor", "timestamp_seconds": 12.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pauses between sentences feel slightly mechanical.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to clearly convey the gentle teasing tone at the end.", "severity": "minor", "timestamp_seconds": 12.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing feels slightly mechanical.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit slightly robotic pacing and limited emotional range." ] } }, { "id": "m17-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-es-ES-A.wav", "model": "s2-pro", "duration": 17.1102 }, "b": { "src": "audio/m17-es-ES-B.wav", "model": "s2.1-pro", "duration": 16.9708 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples feature a strong non-native accent that impacts naturalness, but both remain intelligible and preserve the target text accurately. There is no clear winner between the two.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong non-native accent that impacts naturalness, but both remain intelligible and preserve the target text accurately. There is no clear winner between the two." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy English accent when speaking Spanish, including pronouncing the silent 'h' in 'He reservado'. Neither sounds natural for the requested locale, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' with a heavy English accent.", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' and has a strong English accent throughout.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples feature a strong non-native accent, making it difficult to assess native-level prosody.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "m17-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-fr-FR-A.wav", "model": "s2-pro", "duration": 16.5993 }, "b": { "src": "audio/m17-fr-FR-B.wav", "model": "s2.1-pro", "duration": 16.1581 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples exhibit a severe American accent that heavily distorts the French pronunciation. Sample B is slightly preferred because it approximates the 'é' and 'er' vowel sounds better than Sample A, which pronounces them as 'ee'.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples exhibit a severe American accent that heavily distorts the French pronunciation. Sample B is slightly preferred because it approximates the 'é' and 'er' vowel sounds better than Sample A, which pronounces them as 'ee'." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a heavy foreign accent, but Sample A is preferred. Sample B frequently substitutes the final /e/ sound with /i/ (e.g., 'réservi', 's'instali'), which is highly disruptive to intelligibility." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American accent mispronounces 'é' and 'er' endings as 'ee' (e.g., 'réservé' sounds like 'rezervi').", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces final 'é' or 'er' as 'i' (e.g., 'réservi', 's'instali', 'profiti').", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy American accent affecting overall vowel quality and rhythm.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe American accent distorts many words (e.g., 'deux' sounds like 'doo', 'biscuit' like 'biskwee').", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and ignores the emotional cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy American accent throughout, affecting nasal vowels and general phonetics (e.g., 'temps', 'deux').", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Non-native speaker evaluating the exact severity of the foreign accent.", "Both samples feature a very strong American accent, making it difficult to assess native French prosody." ] } }, { "id": "m17-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.3888 }, "b": { "src": "audio/m17-de-DE-B.wav", "model": "s2-pro", "duration": 19.6876 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred because it has a milder foreign accent and more natural prosody. Sample B suffers from a very heavy, disruptive accent that impacts intelligibility and naturalness.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A is preferred because it has a milder foreign accent and more natural prosody. Sample B suffers from a very heavy, disruptive accent that impacts intelligibility and naturalness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a noticeable foreign accent and make the exact same pronunciation error, reading 'Keks' as the English word 'cakes'. Neither is clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable foreign accent throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Very heavy foreign accent on words like 'Zeit' and 'reserviert'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 17.0 } ] }, "limitations": [ "Evaluator is not a native speaker of all regional German dialects, but the English pronunciation of 'Keks' is a clear error." ] } }, { "id": "m17-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-pt-BR-A.wav", "model": "s2.1-pro", "duration": 19.0838 }, "b": { "src": "audio/m17-pt-BR-B.wav", "model": "s2-pro", "duration": 20.1055 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a strong foreign accent and lack the requested warmth and teasing delivery. Sample A is preferred because Sample B incorrectly pluralizes a word.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples exhibit a strong foreign accent and lack the requested warmth and teasing delivery. Sample A is preferred because Sample B incorrectly pluralizes a word." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a strong foreign accent and robotic prosody. Sample A is slightly more expressive and flows better, despite a minor pluralization error." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, affecting naturalness.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Very disjointed pacing and strong foreign accent.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Flat delivery lacking the requested warmth.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Heard 'noites' instead of expected 'noite'.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Heard 'noites' instead of 'noite'.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stilted pacing and strong foreign accent.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent." ] } }, { "id": "m17-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m17-ar-A.wav", "model": "s2.1-pro", "duration": 18.9445 }, "b": { "src": "audio/m17-ar-B.wav", "model": "s2-pro", "duration": 21.1737 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit similar pronunciation errors regarding Arabic short vowels and dual forms, resulting in comparable overall quality and intelligibility.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar pronunciation errors regarding Arabic short vowels and dual forms, resulting in comparable overall quality and intelligibility." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly better pacing and a closer attempt at the requested expressiveness, whereas Sample B is more monotonous. Both exhibit robotic prosody." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'مقعدين' (maq'adayn) incorrectly, sounding like 'muqa'idin'.", "severity": "major", "timestamp_seconds": 2.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'قرب' (qurba) as 'qarab'.", "severity": "minor", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth and teasing tones.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'قرب' is mispronounced.", "severity": "minor", "timestamp_seconds": 7.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'حجزت' (hajaztu) as 'hajazat' (she booked).", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'مقعدين' incorrectly, similar to sample A.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat disjointed with unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": 0.07 }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'قرب' is pronounced with a non-standard consonant sound.", "severity": "minor", "timestamp_seconds": 8.0 } ] }, "limitations": [ "Both models struggle with correct Arabic short vowel and dual form pronunciations, common in unvowelized text.", "Evaluation of subtle Arabic dialectal vs. MSA pronunciation nuances." ] } } ] }, { "id": "m18", "name": "Pierrot", "slug": "pierrotobbsesivekindmonsterdating-user-3440e1", "gender": "Male", "rank": 18, "portrait": "portraits/m18.webp", "language": "8 languages", "popularity": { "callers": 5, "calls": 6 }, "pairs": [ { "id": "m18-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-zh-CN-A.wav", "model": "s2.1-pro", "duration": 19.1535 }, "b": { "src": "audio/m18-zh-CN-B.wav", "model": "s2-pro", "duration": 20.8253 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are very similar in voice and delivery style. Sample A is slightly preferred due to more natural phrasing and pacing, whereas Sample B has slightly more disjointed pauses.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples are very similar in voice and delivery style. Sample A is slightly preferred due to more natural phrasing and pacing, whereas Sample B has slightly more disjointed pauses." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear audio quality. They both struggle slightly to convey the shift to gentle teasing, resulting in a similar overall performance." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The final sentence lacks the requested teasing tone.", "severity": "minor", "timestamp_seconds": 14.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly disjointed pacing with unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing at the end is not very distinct.", "severity": "minor", "timestamp_seconds": 15.0 } ] }, "limitations": [ "Both samples use a highly breathy, ASMR-like vocal style which may not be universally preferred, but fits the 'warm, reassuring' prompt.", "I am not a native speaker of Mandarin Chinese, but I can assess general pronunciation, text fidelity, and audio quality." ] } }, { "id": "m18-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-ja-JP-A.wav", "model": "s2-pro", "duration": 24.169 }, "b": { "src": "audio/m18-ja-JP-B.wav", "model": "s2.1-pro", "duration": 22.6597 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural and expressive delivery, capturing the warm and reassuring tone better. Sample B sounds slightly more robotic and has minor audio artifacts.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural and expressive delivery, capturing the warm and reassuring tone better. Sample B sounds slightly more robotic and has minor audio artifacts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B exhibits slightly more natural pacing and phrasing compared to Sample A, making the delivery feel a bit more conversational, though both are very similar." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly stiff and less natural compared to A.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing in some phrases.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Japanese speaker, but I can evaluate general pronunciation, prosody, and text fidelity." ] } }, { "id": "m18-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-ko-KR-A.wav", "model": "s2.1-pro", "duration": 21.313 }, "b": { "src": "audio/m18-ko-KR-B.wav", "model": "s2-pro", "duration": 23.5885 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred because it has fewer distracting breath noises. Sample B includes a very prominent and unnatural breath sound that disrupts the flow.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A is preferred because it has fewer distracting breath noises. Sample B includes a very prominent and unnatural breath sound that disrupts the flow." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples adopt an overly breathy, whispered tone that sounds somewhat unnatural for the context. However, Sample B has better pacing and fewer distracting breath artifacts than Sample A, making it slightly more natural." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is somewhat slow and sleepy rather than naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and whispered, which feels slightly unnatural for the intended warm tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "other", "evidence": "Loud, distracting breath noise.", "severity": "major", "timestamp_seconds": 11.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Overly breathy and slow delivery.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pacing with excessively long pauses and loud, distracting breath noises.", "severity": "major", "timestamp_seconds": 11.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The overly breathy delivery sounds more creepy than warm and reassuring.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a highly breathy, ASMR-like vocal style that may not perfectly align with standard conversational expectations." ] } }, { "id": "m18-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-es-ES-A.wav", "model": "s2-pro", "duration": 21.847 }, "b": { "src": "audio/m18-es-ES-B.wav", "model": "s2.1-pro", "duration": 18.2479 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides clearer audio quality and better intelligibility. Sample B sounds somewhat muffled with slight distortion. Both samples feature an overly breathy delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides clearer audio quality and better intelligibility. Sample B sounds somewhat muffled with slight distortion. Both samples feature an overly breathy delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples have a noticeable non-native accent and a breathy delivery. Sample A is preferred because Sample B contains a highly disruptive, exaggerated breath intake before the final sentence." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is excessively breathy and whisper-like, which feels exaggerated for the requested warm tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native (English) accent throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "other", "evidence": "Unnaturally loud and disruptive breath intake.", "severity": "major", "timestamp_seconds": 16.5 } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "The audio sounds muffled with slight background noise and distortion.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice is excessively breathy, sounding more like a whisper than a warm conversation.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native (English) accent throughout, affecting 'prisa' and 'reservado'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a strong non-native (English) accent for Spanish." ] } }, { "id": "m18-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-fr-FR-A.wav", "model": "s2.1-pro", "duration": 18.6195 }, "b": { "src": "audio/m18-fr-FR-B.wav", "model": "s2-pro", "duration": 19.4554 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a strong non-native accent and an overly breathy, whispering delivery that detracts from naturalness. Neither is clearly superior.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent and an overly breathy, whispering delivery that detracts from naturalness. Neither is clearly superior." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples suffer from a heavy non-native accent and disjointed prosody. Sample B is slightly preferred as it is less exaggeratedly breathy than Sample A." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and whispering instead of naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slow and disjointed pacing.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and whispering instead of naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Extremely slow, disjointed, and breathy delivery.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a heavy non-native accent.", "The delivery is unnaturally breathy and whispering.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level nuances." ] } }, { "id": "m18-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.5978 }, "b": { "src": "audio/m18-de-DE-B.wav", "model": "s2-pro", "duration": 23.0545 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A delivers the text smoothly with appropriate warmth. Sample B suffers from an awkward pause and unnatural phrasing when pronouncing the time.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A delivers the text smoothly with appropriate warmth. Sample B suffers from an awkward pause and unnatural phrasing when pronouncing the time." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural cadence and smoother delivery of the time. Sample A has an awkward pause in the middle of the number 'fünfundvierzig'." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Awkward pause and phrasing during '19 Uhr 5... und 40'.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Awkward pause in the middle of the number 'fünfundvierzig'.", "severity": "minor", "timestamp_seconds": 5.0 } ] }, "limitations": [ "I am not a native German speaker, but the number pronunciation error in B is clear." ] } }, { "id": "m18-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-pt-BR-A.wav", "model": "s2.1-pro", "duration": 20.5235 }, "b": { "src": "audio/m18-pt-BR-B.wav", "model": "s2-pro", "duration": 22.9848 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text with a similar whispering, breathy tone that fits the warm direction but introduces noticeable background noise. Neither has a clear advantage.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with a similar whispering, breathy tone that fits the warm direction but introduces noticeable background noise. Neither has a clear advantage." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from severe robotic prosody and strong foreign accents. Sample B is slightly preferred for being marginally more fluid than A's highly disjointed delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and breath noise throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural, mechanical rhythm and intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign accent impacting vowel sounds.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and breath noise throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly disjointed, slow, and unnatural cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects multiple words, e.g., 'Não', 'Reservei'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a heavy whispering style that introduces significant background hiss.", "Both samples exhibit heavy non-native accents and robotic prosody." ] } }, { "id": "m18-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m18-ar-A.wav", "model": "s2.1-pro", "duration": 23.0312 }, "b": { "src": "audio/m18-ar-B.wav", "model": "s2-pro", "duration": 25.6319 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from an overly whispered delivery and unnatural pauses. Sample A is slightly preferred due to more accurate pronunciation of the first-person verb 'حجزت' (I booked), whereas B pronounces it as third-person feminine.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from an overly whispered delivery and unnatural pauses. Sample A is slightly preferred due to more accurate pronunciation of the first-person verb 'حجزت' (I booked), whereas B pronounces it as third-person feminine." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples employ an overly breathy, whispered delivery with unnatural mid-phrase pauses. Neither clearly outperforms the other." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is excessively whispered, resembling ASMR rather than a natural warm conversation.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 9.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an exaggerated whisper rather than a natural warm tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Excessively whispered, ASMR-style delivery.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'حجزت' as third-person feminine (hajazat) instead of first-person (hajaztu).", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurba' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an exaggerated whisper rather than a natural warm tone.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature an exaggerated, ASMR-like whispered delivery that makes precise phonetic evaluation difficult.", "Both samples use an exaggerated whispered delivery that may not fit a standard conversation.", "Both samples contain unnatural pauses." ] } } ] }, { "id": "m19", "name": "Ethan", "slug": "ethan-81cb00", "gender": "Male", "rank": 19, "portrait": "portraits/m19.webp", "language": "8 languages", "popularity": { "callers": 5, "calls": 5 }, "pairs": [ { "id": "m19-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-zh-CN-A.wav", "model": "s2.1-pro", "duration": 14.6488 }, "b": { "src": "audio/m19-zh-CN-B.wav", "model": "s2-pro", "duration": 14.9971 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers smoother phrasing and a more natural delivery that better captures the intended warm and teasing tones. Sample A sounds somewhat robotic and disjointed.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers smoother phrasing and a more natural delivery that better captures the intended warm and teasing tones. Sample A sounds somewhat robotic and disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers more natural prosody and better captures the warm and gently teasing tone. Sample A sounds slightly more robotic and flat." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The phrasing is somewhat choppy and lacks natural flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat and mechanical delivery throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "m19-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-ja-JP-A.wav", "model": "s2.1-pro", "duration": 18.3176 }, "b": { "src": "audio/m19-ja-JP-B.wav", "model": "s2-pro", "duration": 17.4817 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A includes a natural-sounding chuckle that perfectly captures the 'gently teasing' direction. Sample B is also good but slightly less expressive in the second half.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A includes a natural-sounding chuckle that perfectly captures the 'gently teasing' direction. Sample B is also good but slightly less expressive in the second half." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B includes a natural-sounding chuckle that perfectly captures the 'gently teasing' direction. Sample A is overly breathy and whispery throughout." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is overly whispery throughout, lacking dynamic range for the teasing section.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an extremely breathy, whispered delivery which may not be universally preferred, but fits the intimate tone.", "Evaluator is not a native Japanese speaker, but can assess general pronunciation and prosody." ] } }, { "id": "m19-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-ko-KR-A.wav", "model": "s2-pro", "duration": 16.0188 }, "b": { "src": "audio/m19-ko-KR-B.wav", "model": "s2.1-pro", "duration": 16.1349 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more standard and usable delivery. Sample A relies on an excessively breathy, whispered tone that feels unnatural for the context.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B provides a more standard and usable delivery. Sample A relies on an excessively breathy, whispered tone that feels unnatural for the context." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A provides a more balanced delivery. Sample B is excessively breathy, which feels unnatural for the requested warm and teasing tone." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is heavily whispered rather than just warm and reassuring.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and ASMR-like, which does not fit the gentle teasing direction.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are slightly stiff and lack dynamic range.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing and pauses between sentences.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, but can assess general prosody and text fidelity.", "Evaluation of subtle Korean emotional nuances is based on general prosodic cues." ] } }, { "id": "m19-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.9739 }, "b": { "src": "audio/m19-es-ES-B.wav", "model": "s2-pro", "duration": 15.1133 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from a heavy non-native accent. Sample B is slightly preferred for having marginally better prosody and expressiveness compared to the robotic delivery of Sample A.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a heavy non-native accent. Sample B is slightly preferred for having marginally better prosody and expressiveness compared to the robotic delivery of Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is slightly more expressive and has better phrasing than Sample B, although both suffer from a heavy foreign accent." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly noticeable on 'r' sounds (e.g., 'prisa', 'reservado').", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is disjointed and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warmth.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting the pronunciation of 'r' and vowel sounds.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native (American English) accent, which affects the naturalness and pronunciation for the requested es-ES locale.", "Both samples exhibit a very strong English accent when speaking Spanish, which affects naturalness." ] } }, { "id": "m19-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-fr-FR-A.wav", "model": "s2-pro", "duration": 14.4399 }, "b": { "src": "audio/m19-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.138 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a heavy foreign accent that distorts the pronunciation of many French words. Neither is suitable for a native French locale, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy foreign accent that distorts the pronunciation of many French words. Neither is suitable for a native French locale, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural delivery with better pacing and pronunciation. Sample A sounds slightly more robotic and mispronounces the first word." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Prends' sounds like 'Prance' and 'réservé' uses an English 'r'.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Prends' sounds like 'Prins' and 'fenêtre' uses an English 'r'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'Prends' as 'Prins'.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation sound slightly stilted.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong foreign (likely English) accent, significantly impacting the naturalness of the French pronunciation." ] } }, { "id": "m19-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-de-DE-A.wav", "model": "s2-pro", "duration": 15.2525 }, "b": { "src": "audio/m19-de-DE-B.wav", "model": "s2.1-pro", "duration": 13.9522 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a noticeable English accent, particularly on words like 'reden' and 'Keks'. Sample A is preferred for having slightly more natural prosody and better alignment with the requested warm delivery, whereas Sample B is very flat.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples exhibit a noticeable English accent, particularly on words like 'reden' and 'Keks'. Sample A is preferred for having slightly more natural prosody and better alignment with the requested warm delivery, whereas Sample B is very flat." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A has slightly better phrasing and flow compared to Sample B, which sounds a bit more disjointed. Both have noticeable non-native accents." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 13.5 }, { "order": "AB", "category": "pronunciation", "evidence": "English 'r' sound in 'reden'.", "severity": "minor", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is somewhat choppy and disjointed.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 12.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is very flat and lacks the requested warmth.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent on several words like 'reden' and 'Keks'.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a noticeable non-native accent." ] } }, { "id": "m19-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-pt-BR-A.wav", "model": "s2-pro", "duration": 17.412 }, "b": { "src": "audio/m19-pt-BR-B.wav", "model": "s2.1-pro", "duration": 16.5529 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese pronunciation. Sample B is preferred for its clearer audio quality, whereas Sample A sounds muffled with noticeable background hiss.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese pronunciation. Sample B is preferred for its clearer audio quality, whereas Sample A sounds muffled with noticeable background hiss." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese pronunciation. Sample A has choppy, robotic prosody, while Sample B is excessively breathy and whispered. Neither is clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "wrong_language", "evidence": "Pronunciation and vowel reduction strongly indicate European Portuguese (pt-PT) rather than the requested Brazilian Portuguese (pt-BR).", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "The audio sounds muffled with a persistent background hiss.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Speaker uses European Portuguese pronunciation (e.g., 'dia', 'noite', 'juntos') instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is excessively breathy and whispered, which feels unnatural for the requested warm and reassuring tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "wrong_language", "evidence": "Pronunciation and vowel reduction strongly indicate European Portuguese (pt-PT) rather than the requested Brazilian Portuguese (pt-BR).", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat rushed and upbeat, missing the requested 'warm, reassuring' tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Speaker uses European Portuguese pronunciation (e.g., 'dia', 'noite', 'juntos') instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is choppy and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a European Portuguese (pt-PT) accent rather than the requested Brazilian Portuguese (pt-BR) locale.", "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "m19-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m19-ar-A.wav", "model": "s2-pro", "duration": 18.6427 }, "b": { "src": "audio/m19-ar-B.wav", "model": "s2.1-pro", "duration": 17.7835 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred because it maintains better text fidelity. Sample B substitutes a word, altering the intended text slightly, and both share similar prosodic limitations.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because it maintains better text fidelity. Sample B substitutes a word, altering the intended text slightly, and both share similar prosodic limitations." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a more natural and appropriate delivery for the context. Sample B relies on an unnatural whisper that slightly impairs intelligibility and sounds forced." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly disjointed phrasing and unnatural pauses throughout the delivery.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speaker uses a forced whisper throughout, which does not fit the intended warm delivery.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Muffled audio quality due to the whispering style.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Speaker says 'قريب' instead of the expected 'قرب'.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is somewhat uneven with unnatural pauses.", "severity": "minor", "timestamp_seconds": 10.0 }, { "order": "BA", "category": "distortion", "evidence": "Slightly muffled audio quality with minor background hiss.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of Arabic prosody and exact vowel endings (tashkeel) is based on general Modern Standard Arabic conventions.", "Evaluation of Arabic pronunciation nuances is based on general Modern Standard Arabic conventions." ] } } ] }, { "id": "m20", "name": "Jae-won", "slug": "jae-won-3ed8a6", "gender": "Male", "rank": 20, "portrait": "portraits/m20.webp", "language": "8 languages", "popularity": { "callers": 5, "calls": 5 }, "pairs": [ { "id": "m20-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-zh-CN-A.wav", "model": "s2-pro", "duration": 14.0916 }, "b": { "src": "audio/m20-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.0885 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has noticeable audio quality issues but flows slightly more naturally than Sample B, which suffers from highly robotic and disjointed prosody.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has noticeable audio quality issues but flows slightly more naturally than Sample B, which suffers from highly robotic and disjointed prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery that better aligns with the requested warm and teasing tones. Sample A suffers from robotic prosody and a flat delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_substitution", "evidence": "Expected '不过先说好', heard '如果先说好'.", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and muffled audio quality throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Minor synthetic artifacts in the pacing.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Expected '不过先说好', heard '如果先说好'.", "severity": "major", "timestamp_seconds": 12.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Speech is highly disjointed with unnatural pauses and flat intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and disjointed, lacking natural flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a text substitution error, reading '如果' instead of '不过'." ] } }, { "id": "m20-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-ja-JP-A.wav", "model": "s2.1-pro", "duration": 17.7139 }, "b": { "src": "audio/m20-ja-JP-B.wav", "model": "s2-pro", "duration": 16.6457 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with appropriate emotion. Both suffer from minor audio quality and pacing issues, making them roughly equivalent in overall quality.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with appropriate emotion. Both suffer from minor audio quality and pacing issues, making them roughly equivalent in overall quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural delivery with better audio quality. Sample B has noticeable reverb and slightly less natural prosody." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Slightly muffled audio quality with background hiss.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable room reverb throughout the recording.", "severity": "minor", "timestamp_seconds": 0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Some unnatural pauses and slight synthetic artifacts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Japanese, but can evaluate general pronunciation and prosody.", "Evaluated based on general Japanese pronunciation and prosody." ] } }, { "id": "m20-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-ko-KR-A.wav", "model": "s2-pro", "duration": 15.9027 }, "b": { "src": "audio/m20-ko-KR-B.wav", "model": "s2.1-pro", "duration": 16.3671 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B has noticeably better audio quality with less background noise. It also features a slightly more natural pacing and delivery compared to Sample A.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B has noticeably better audio quality with less background noise. It also features a slightly more natural pacing and delivery compared to Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B captures the requested warm and teasing delivery much better than Sample A, which sounds flat and robotic. Both samples have good text fidelity and intelligibility." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Constant background hiss and static throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and monotonous throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle nuances in accent or regional variations may not be fully captured.", "I am not a native Korean speaker, but I can evaluate general prosody, emotion, and intelligibility." ] } }, { "id": "m20-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.347 }, "b": { "src": "audio/m20-es-ES-B.wav", "model": "s2-pro", "duration": 13.604 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic vocoder artifacts and a heavy English accent that degrades the Spanish pronunciation. Neither is usable.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic vocoder artifacts and a heavy English accent that degrades the Spanish pronunciation. Neither is usable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples suffer from a strong foreign accent. Sample B is slightly more fluid and natural in its phrasing, whereas Sample A is noticeably choppy." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The voice sounds highly synthetic and robotic throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on Spanish words like 'reservado' and 'tarde'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background noise and vocoder distortion.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign accent throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable room echo or reverb.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The voice sounds highly synthetic and robotic throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on Spanish words like 'reservado' and 'tarde'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "distortion", "evidence": "Vocoder distortion is present throughout the speech.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural cadence.", "severity": "major", "timestamp_seconds": 2.0 } ] }, "limitations": [ "Both samples exhibit severe robotic artifacts and heavy non-native accents.", "Both samples feature a strong non-native accent." ] } }, { "id": "m20-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-fr-FR-A.wav", "model": "s2-pro", "duration": 13.3021 }, "b": { "src": "audio/m20-fr-FR-B.wav", "model": "s2.1-pro", "duration": 13.7432 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples feature a heavy English accent when speaking French. Sample B is preferred because it has smoother prosody and fewer unnatural pauses than Sample A.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples feature a heavy English accent when speaking French. Sample B is preferred because it has smoother prosody and fewer unnatural pauses than Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe American accent that heavily impacts French pronunciation. Sample A is slightly preferred because it avoids the overly breathy and whispered delivery found in Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, mispronouncing French Rs and nasal vowels.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy delivery with unnatural pauses between phrases.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe American accent throughout the French text.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is overly breathy and whispered, not fitting the warm or teasing cues.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, mispronouncing French Rs and nasal vowels.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe American accent throughout, e.g., 'Prends ton temps' and 'biscuit'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a severe non-native (English) accent, affecting French pronunciation.", "Both samples feature a severe foreign accent." ] } }, { "id": "m20-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-de-DE-A.wav", "model": "s2-pro", "duration": 14.556 }, "b": { "src": "audio/m20-de-DE-B.wav", "model": "s2.1-pro", "duration": 15.1365 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better audio clarity and a more natural, expressive delivery. Both samples share a pronunciation error on the word 'Keks'.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better audio clarity and a more natural, expressive delivery. Both samples share a pronunciation error on the word 'Keks'." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A is preferred as it maintains better text fidelity. Sample B drops a syllable in 'deinen', and both suffer from a strong foreign accent." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "text_omission", "evidence": "Heard 'dein Tag' instead of 'deinen Tag'.", "severity": "minor", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on words like 'reserviert' and 'Keks'.", "severity": "minor", "timestamp_seconds": 4.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 13.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on words like 'reserviert' and 'Keks'.", "severity": "minor", "timestamp_seconds": 3.5 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, but this is not heavily penalized.", "Both samples feature a strong foreign (American) accent, affecting pronunciation of several words." ] } }, { "id": "m20-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-pt-BR-A.wav", "model": "s2.1-pro", "duration": 17.2727 }, "b": { "src": "audio/m20-pt-BR-B.wav", "model": "s2-pro", "duration": 14.2309 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is significantly more natural and expressive, capturing the intended warm and teasing tones well. Sample A suffers from robotic prosody and choppy phrasing.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B is significantly more natural and expressive, capturing the intended warm and teasing tones well. Sample A suffers from robotic prosody and choppy phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred due to better audio quality and slightly more natural pacing. Sample B suffers from noticeable background noise and awkward pauses." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The phrasing is choppy and lacks natural flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and room noise throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Awkward, unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": 3.0 } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Accent sounds more European Portuguese than Brazilian Portuguese in words like 'pressa' and 'biscoito'.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit accents that lean towards European Portuguese or non-native pronunciation rather than standard Brazilian Portuguese." ] } }, { "id": "m20-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/m20-ar-A.wav", "model": "s2-pro", "duration": 18.4801 }, "b": { "src": "audio/m20-ar-B.wav", "model": "s2.1-pro", "duration": 19.2232 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from poor audio quality and unnatural prosody. Sample A is slightly preferred for having marginally better expressiveness and a less robotic delivery than Sample B.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples suffer from poor audio quality and unnatural prosody. Sample A is slightly preferred for having marginally better expressiveness and a less robotic delivery than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and clearer articulation than Sample A. Both have minor audio quality issues, but B's pacing and expressiveness better match the intended delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause before 'an-nafitha'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and muffled vocal quality throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Minor background noise present.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is stiff and lacks the requested warmth and teasing tone.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Background hiss and slightly synthetic vocal texture.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and slightly muffled audio quality throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and phrasing in some sentences.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit noticeable background hiss and muffled audio quality.", "Prosody and pacing are unnatural in both samples.", "Evaluation of Arabic pronunciation nuances is based on general MSA standards." ] } } ] }, { "id": "f01", "name": "Mariam", "slug": "mariam-30bd41", "gender": "Female", "rank": 1, "portrait": "portraits/f01.webp", "language": "8 languages", "popularity": { "callers": 325, "calls": 426 }, "pairs": [ { "id": "f01-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.3903 }, "b": { "src": "audio/f01-zh-CN-B.wav", "model": "s2-pro", "duration": 16.5761 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better audio quality and a more natural, expressive delivery. Sample A sounds a bit more muffled and has minor robotic artifacts in its prosody.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly better audio quality and a more natural, expressive delivery. Sample A sounds a bit more muffled and has minor robotic artifacts in its prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better naturalness and expressiveness, capturing the warm and teasing tones more effectively than Sample B, which sounds a bit flatter." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing and muffled tone throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [ "None", "Evaluator is not a native speaker of all regional Mandarin accents." ] } }, { "id": "f01-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-ja-JP-A.wav", "model": "s2.1-pro", "duration": 18.6891 }, "b": { "src": "audio/f01-ja-JP-B.wav", "model": "s2-pro", "duration": 18.9677 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural and expressive delivery that better captures the intended warmth and gentle teasing. Sample B is clear but slightly more robotic in its pacing.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better captures the intended warmth and gentle teasing. Sample B is clear but slightly more robotic in its pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A is slightly more intelligible and articulates the words more clearly. Sample B slurs some pronunciation, making it harder to understand in parts." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "pronunciation", "evidence": "'Ashita' is slurred and sounds like 'Asa'.", "severity": "major", "timestamp_seconds": 2.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is slightly slow and deliberate, reducing naturalness.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "'Ashita' is pronounced slightly like 'Asuta'.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat intonation throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, which affects naturalness." ] } }, { "id": "f01-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-ko-KR-A.wav", "model": "s2.1-pro", "duration": 17.2495 }, "b": { "src": "audio/f01-ko-KR-B.wav", "model": "s2-pro", "duration": 18.4569 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers slightly better pacing and a more natural transition into the teasing tone at the end. Both samples are highly intelligible and accurate.", "confidence": 0.65, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly better pacing and a more natural transition into the teasing tone at the end. Both samples are highly intelligible and accurate." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Sample A captures the requested emotional shift slightly better than Sample B, which remains somewhat flat. Both have acceptable intelligibility and text fidelity." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery remains relatively flat and does not clearly shift to gently teasing.", "severity": "minor", "timestamp_seconds": 12.0 } ], "B": [] }, "limitations": [ "I am not a native Korean speaker, so subtle nuances in accent or regional pronunciation may not be fully evaluated." ] } }, { "id": "f01-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-es-ES-A.wav", "model": "s2.1-pro", "duration": 16.9708 }, "b": { "src": "audio/f01-es-ES-B.wav", "model": "s2-pro", "duration": 17.621 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a heavy non-native accent that significantly impacts naturalness and pronunciation. They are equally intelligible and follow the text accurately, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy non-native accent that significantly impacts naturalness and pronunciation. They are equally intelligible and follow the text accurately, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy non-native accent that distorts pronunciation and detracts from the intended delivery. Neither is clearly superior." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'siete y cuarenta y cinco' pronounced with incorrect vowels.", "severity": "major", "timestamp_seconds": 4.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'siete y cuarenta y cinco' pronounced with incorrect vowels.", "severity": "major", "timestamp_seconds": 4.0 } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody.", "Both samples feature a very strong non-native (English) accent that significantly affects the pronunciation of Spanish phonemes." ] } }, { "id": "f01-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-fr-FR-A.wav", "model": "s2-pro", "duration": 15.9027 }, "b": { "src": "audio/f01-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.6488 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from severe pronunciation issues and a heavy non-native accent that significantly degrades intelligibility and naturalness. Neither is clearly better.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe pronunciation issues and a heavy non-native accent that significantly degrades intelligibility and naturalness. Neither is clearly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent that heavily impacts the naturalness and pronunciation of the French text. Neither sample provides a convincing or native-sounding delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Prends ton temps' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Prends ton temps' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a very heavy non-native accent, making it difficult to assess subtle prosodic features.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f01-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.4585 }, "b": { "src": "audio/f01-de-DE-B.wav", "model": "s2-pro", "duration": 17.7139 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe English accent that makes the German pronunciation sound unnatural. They are tied in quality and text fidelity.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent that makes the German pronunciation sound unnatural. They are tied in quality and text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy English accent that significantly impacts the pronunciation of the German text. Neither sample is clearly superior in naturalness or intelligibility." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir Zeit' sounds like 'Last dear site'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Lass dir Zeit' sounds like 'Last dear site'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Keks' pronounced like English 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ] }, "limitations": [ "Both samples feature a very strong non-native (English) accent that significantly impacts pronunciation.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level naturalness." ] } }, { "id": "f01-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-pt-BR-A.wav", "model": "s2.1-pro", "duration": 18.5034 }, "b": { "src": "audio/f01-pt-BR-B.wav", "model": "s2-pro", "duration": 18.0622 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to use the requested Brazilian Portuguese locale, instead sounding distinctly European Portuguese. They also share similar robotic pacing and flat delivery, making it a tie.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, instead sounding distinctly European Portuguese. They also share similar robotic pacing and flat delivery, making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B provides slightly clearer audio quality and a more natural expressive delivery that better fits the requested warm and teasing tones." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent and phonology instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is somewhat unnatural with disjointed pauses between phrases.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Some vowels and consonants sound slightly non-native for Brazilian Portuguese.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent rather than the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The accent leans towards European Portuguese in several words, which is unexpected for the requested pt-BR locale.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested dynamic shifts in tone.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale.", "Both samples exhibit some non-native or mixed accent characteristics for Brazilian Portuguese." ] } }, { "id": "f01-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f01-ar-A.wav", "model": "s2.1-pro", "duration": 19.2 }, "b": { "src": "audio/f01-ar-B.wav", "model": "s2-pro", "duration": 19.6411 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples sound nearly identical, featuring flat prosody and slightly muffled audio. Neither effectively captures the requested warm or teasing emotional shifts, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples sound nearly identical, featuring flat prosody and slightly muffled audio. Neither effectively captures the requested warm or teasing emotional shifts, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples provide a similar level of quality, with comparable pronunciation and pacing. Neither sample stands out significantly in expressiveness or naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and gently teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech sounds mechanical with unnatural pacing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not reflect the warm or teasing directions.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ] }, "limitations": [ "Both samples exhibit a similar flat, slightly robotic delivery that makes it difficult to distinguish meaningful differences in expressiveness.", "Evaluation of Arabic pronunciation nuances is limited by the provided audio quality." ] } } ] }, { "id": "f02", "name": "Rin", "slug": "rin-aoki", "gender": "Female", "rank": 2, "portrait": "portraits/f02.webp", "language": "8 languages", "popularity": { "callers": 318, "calls": 412 }, "pairs": [ { "id": "f02-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-zh-CN-A.wav", "model": "s2-pro", "duration": 17.0405 }, "b": { "src": "audio/f02-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.3903 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples mispronounce '来' as 'gai'. However, Sample A also severely mispronounces '不着急', making Sample B the better choice due to higher text fidelity.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples mispronounce '来' as 'gai'. However, Sample A also severely mispronounces '不着急', making Sample B the better choice due to higher text fidelity." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples fail to pronounce '慢慢来' correctly, substituting 'gai' for 'lai'. Sample A is slightly preferred as its mispronunciation of '不着急' is closer to the target than Sample B's." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "2.5/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '慢慢来' as 'man man gai'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '不着急' as 'fu zhou ji'.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '慢慢来' as 'man man gai'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '不着急' as 'fu zhou ji'.", "severity": "major", "timestamp_seconds": 1.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '慢慢来' as 'man man gai'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '慢慢来' as 'man man gai'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '不着急' as 'bu jiao ji'.", "severity": "major", "timestamp_seconds": 1.0 } ] }, "limitations": [ "Both models exhibit a specific pronunciation artifact on the word '来'.", "Both models exhibit severe pronunciation errors on the first few words." ] } }, { "id": "f02-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-ja-JP-A.wav", "model": "s2.1-pro", "duration": 17.6442 }, "b": { "src": "audio/f02-ja-JP-B.wav", "model": "s2-pro", "duration": 16.2975 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B is slightly flatter and less dynamic.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B is slightly flatter and less dynamic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B successfully captures the requested warm and teasing tones, sounding much more natural and expressive. Sample A is relatively flat and robotic in its delivery." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f02-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-ko-KR-A.wav", "model": "s2-pro", "duration": 17.2959 }, "b": { "src": "audio/f02-ko-KR-B.wav", "model": "s2.1-pro", "duration": 17.412 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural prosody and better captures the gentle teasing tone requested in the prompt. Sample A is acceptable but a bit flatter in delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly more natural prosody and better captures the gentle teasing tone requested in the prompt. Sample A is acceptable but a bit flatter in delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better matches the requested warm and gently teasing tones. Sample B is somewhat flat and robotic in comparison." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The teasing tone at the end is somewhat subtle and flat.", "severity": "minor", "timestamp_seconds": 12.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [ "I am not a native Korean speaker, but I can evaluate general prosody and text fidelity." ] } }, { "id": "f02-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-es-ES-A.wav", "model": "s2-pro", "duration": 14.5328 }, "b": { "src": "audio/f02-es-ES-B.wav", "model": "s2.1-pro", "duration": 17.4817 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly better phrasing and flow. Both samples struggle with the requested emotional shifts and exhibit somewhat robotic prosody, but A is less disjointed.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly better phrasing and flow. Both samples struggle with the requested emotional shifts and exhibit somewhat robotic prosody, but A is less disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and flow. Sample A sounds a bit more robotic in its cadence." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is relatively flat and does not clearly convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not very pronounced.", "severity": "minor", "timestamp_seconds": 11.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is disjointed and lacks natural cadence, ignoring the emotional cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation feel somewhat mechanical throughout the sample.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples lack strong regional markers for Spain Spanish, but are evaluated primarily on general intelligibility and prosody." ] } }, { "id": "f02-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-fr-FR-A.wav", "model": "s2-pro", "duration": 13.8361 }, "b": { "src": "audio/f02-fr-FR-B.wav", "model": "s2.1-pro", "duration": 15.3687 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly more natural phrasing and flow than Sample A, which suffers from highly robotic and choppy prosody. However, both fail to capture the requested warm and reassuring tone.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly more natural phrasing and flow than Sample A, which suffers from highly robotic and choppy prosody. However, both fail to capture the requested warm and reassuring tone." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural flow and phrasing compared to Sample A, which has some robotic prosody and unnatural pauses. Both samples have perfect text fidelity." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Frequent unnatural pauses and choppy phrasing throughout the utterance.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, missing both the warm and teasing cues.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The tone is loud and slightly aggressive, failing to sound warm or reassuring.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses and cadence throughout the utterance.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples struggle significantly with the requested emotional delivery.", "Evaluation of subtle regional accents is limited." ] } }, { "id": "f02-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.7835 }, "b": { "src": "audio/f02-de-DE-B.wav", "model": "s2-pro", "duration": 16.0652 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from robotic prosody and a noticeable non-native accent. Neither successfully captures the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from robotic prosody and a noticeable non-native accent. Neither successfully captures the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and native-sounding delivery, although slightly flat. Sample B has a strong non-native accent and choppy prosody that detracts from the overall quality." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is disjointed with unnatural pauses.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "pronunciation", "evidence": "'Keks' is pronounced with an English-like vowel.", "severity": "minor", "timestamp_seconds": 16.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects multiple words, such as 'reserviert' and 'genießen'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The rhythm is choppy with unnatural pauses between words.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Expected 'Lass', heard 'Lasse'.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks natural cadence.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "pronunciation", "evidence": "'Keks' is pronounced with an English-like vowel.", "severity": "minor", "timestamp_seconds": 15.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, particularly on the word 'Keks'." ] } }, { "id": "f02-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-pt-BR-A.wav", "model": "s2-pro", "duration": 15.5544 }, "b": { "src": "audio/f02-pt-BR-B.wav", "model": "s2.1-pro", "duration": 18.0158 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to provide a native Brazilian Portuguese accent. Sample A uses a European Portuguese accent, while Sample B has a heavy, non-native Spanish accent. Sample A is preferred for sounding like native Portuguese, despite the locale mismatch.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to provide a native Brazilian Portuguese accent. Sample A uses a European Portuguese accent, while Sample B has a heavy, non-native Spanish accent. Sample A is preferred for sounding like native Portuguese, despite the locale mismatch." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, instead sounding like European Portuguese. They are similarly intelligible but lack the requested expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent rather than the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The accent sounds distinctly European Portuguese rather than Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and misses the warm and teasing cues.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker has a strong, non-native Spanish accent when speaking Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The accent sounds distinctly European Portuguese rather than Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and gently teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Brazilian Portuguese, but I can detect major accent deviations.", "Both samples exhibit a European Portuguese accent rather than the requested Brazilian Portuguese." ] } }, { "id": "f02-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f02-ar-A.wav", "model": "s2.1-pro", "duration": 18.3176 }, "b": { "src": "audio/f02-ar-B.wav", "model": "s2-pro", "duration": 19.2696 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit similar robotic prosody and fail to capture the requested emotional shifts. Intelligibility and text fidelity are comparable.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar robotic prosody and fail to capture the requested emotional shifts. Intelligibility and text fidelity are comparable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit highly robotic prosody and unnatural phrasing. Neither successfully conveys the requested warm or teasing delivery, making them equally limited." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is unnatural and mechanical throughout the utterance.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warmth and teasing tone, sounding entirely neutral and synthetic.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The phrasing is disjointed with unnatural pauses, such as between 'qurb' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and synthetic, failing to capture the warm or teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, but can assess general prosody and text fidelity.", "I am not a native speaker of Modern Standard Arabic, but the robotic prosody is clearly audible." ] } } ] }, { "id": "f03", "name": "Antionette trella vandersloot", "slug": "antionette-trella-vandersloot-952e8b", "gender": "Female", "rank": 3, "portrait": "portraits/f03.webp", "language": "8 languages", "popularity": { "callers": 176, "calls": 223 }, "pairs": [ { "id": "f03-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.8547 }, "b": { "src": "audio/f03-zh-CN-B.wav", "model": "s2-pro", "duration": 17.3656 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers more natural phrasing and better captures the warm, reassuring tone. Sample A sounds slightly more robotic in its pacing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers more natural phrasing and better captures the warm, reassuring tone. Sample A sounds slightly more robotic in its pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers more natural prosody and better captures the requested warm and teasing tones. Sample B is slightly stiffer in its delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is somewhat rigid and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Slight background hiss present throughout the recording.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit some background hiss and lower audio fidelity.", "Evaluator is not a native speaker of all regional Mandarin accents." ] } }, { "id": "f03-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-ja-JP-A.wav", "model": "s2.1-pro", "duration": 22.0096 }, "b": { "src": "audio/f03-ja-JP-B.wav", "model": "s2-pro", "duration": 19.1535 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples mispronounce '明日の夜' (ashita no yoru). Sample A says 'asa no yoru', while Sample B says 'asutaro yoru'. Sample A is preferred for a more natural, conversational delivery, whereas B relies on an exaggerated whisper.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples mispronounce '明日の夜' (ashita no yoru). Sample A says 'asa no yoru', while Sample B says 'asutaro yoru'. Sample A is preferred for a more natural, conversational delivery, whereas B relies on an exaggerated whisper." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples struggle with basic Japanese pronunciation, mispronouncing 'isoganakute' and 'ashita no yoru'. Sample B is slightly more expressive, but neither provides a fully accurate reading." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 4, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日の夜' (ashita no yoru) as 'asa no yoru'.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'isoganakute' as 'isaganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'ashita no yoru' as 'asa no yoru'.", "severity": "major", "timestamp_seconds": 3.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日の夜' (ashita no yoru) as 'asutaro yoru'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'isoganakute' as 'isaganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'ashita no yoru' as 'asutaro yoru'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "I am not a native Japanese speaker, but I can identify clear pronunciation errors.", "Both samples exhibit pronunciation errors on common Japanese words, suggesting underlying model limitations with the language." ] } }, { "id": "f03-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-ko-KR-A.wav", "model": "s2.1-pro", "duration": 17.6674 }, "b": { "src": "audio/f03-ko-KR-B.wav", "model": "s2-pro", "duration": 17.8532 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural and expressive delivery, capturing the warm and reassuring tone better than A. Sample A has some minor robotic prosody.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural and expressive delivery, capturing the warm and reassuring tone better than A. Sample A has some minor robotic prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly more natural prosody and phrasing. Sample A sounds a bit more robotic and stiff. Both have minor background noise." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural phrasing and cadence in the middle sentences.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "other", "evidence": "Minor background noise/hiss present throughout the recording.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so my evaluation of subtle prosodic naturalness is limited." ] } }, { "id": "f03-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-es-ES-A.wav", "model": "s2.1-pro", "duration": 16.9012 }, "b": { "src": "audio/f03-es-ES-B.wav", "model": "s2-pro", "duration": 16.4135 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a heavy English accent that significantly detracts from the naturalness of the Spanish pronunciation. They are tied in overall quality.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a heavy English accent that significantly detracts from the naturalness of the Spanish pronunciation. They are tied in overall quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples feature a noticeable non-native accent and heavy reverb. Sample A provides slightly better phrasing and expressiveness compared to B's flatter delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'No tengas prisa'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent, particularly on 'r' sounds.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "distortion", "evidence": "Heavy reverb or echo effect throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is somewhat flat and lacks the requested warmth.", "severity": "minor", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'No tengas prisa'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent, particularly on 'r' sounds.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "distortion", "evidence": "Heavy reverb or echo effect throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a very strong non-native accent.", "Cannot evaluate native accent authenticity perfectly." ] } }, { "id": "f03-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-fr-FR-A.wav", "model": "s2.1-pro", "duration": 16.46 }, "b": { "src": "audio/f03-fr-FR-B.wav", "model": "s2-pro", "duration": 15.1597 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe pronunciation errors, applying English phonetics to the French text. Neither produces acceptable French speech.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe pronunciation errors, applying English phonetics to the French text. Neither produces acceptable French speech." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples have a heavy English accent, but Sample A systematically mispronounces 'é' and 'er' endings as 'i', severely impacting intelligibility. Sample B is more understandable despite the accent." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 2, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'temps' and 'tranquille' are heavily mispronounced.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, such as pronouncing 'biscuit' with an English 'w' sound.", "severity": "major", "timestamp_seconds": 14.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'fenêtre' and 'biscuit' are heavily mispronounced.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Systematic mispronunciation of 'é' and 'er' endings as 'i' (e.g., 'réservé' sounds like 'réservi', 'parler' sounds like 'parli').", "severity": "major", "timestamp_seconds": 2.0 } ] }, "limitations": [ "Both samples exhibit a severe non-native accent, applying English phonology to French words.", "Both samples feature a strong non-native (English) accent, making it difficult to assess native fr-FR prosody." ] } }, { "id": "f03-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.8068 }, "b": { "src": "audio/f03-de-DE-B.wav", "model": "s2-pro", "duration": 17.0869 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer audio and correct pronunciation of the time. Sample A has noticeable background hiss and incorrectly adds a suffix to the number '45'.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides clearer audio and correct pronunciation of the time. Sample A has noticeable background hiss and incorrectly adds a suffix to the number '45'." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A is preferred due to slightly better audio quality. Sample B has more noticeable background hiss and a slightly muffled sound, though both share similar non-native pronunciation traits." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 2.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces '45' as 'fünfundvierzigst' instead of 'fünfundvierzig'.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and slightly muffled audio quality.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'reserviert' and 'Fenster'.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native (American) accent, which affects naturalness but not strict text fidelity." ] } }, { "id": "f03-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-pt-BR-A.wav", "model": "s2-pro", "duration": 17.7603 }, "b": { "src": "audio/f03-pt-BR-B.wav", "model": "s2.1-pro", "duration": 19.9894 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples feature a noticeable non-native accent that affects naturalness. Sample A is preferred as it has more stable prosody and better audio quality, whereas Sample B sounds more robotic and includes an unprompted vocalization.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples feature a noticeable non-native accent that affects naturalness. Sample A is preferred as it has more stable prosody and better audio quality, whereas Sample B sounds more robotic and includes an unprompted vocalization." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B provides clearer audio and more consistent pacing. Sample A includes distracting background noise and an unnatural breath artifact." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly mechanical cadence in some phrases.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural cadence and phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "Unprompted laugh or breath sound.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "other", "evidence": "Unnatural breath or laugh artifact.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "distortion", "evidence": "Background hiss and slightly muffled audio.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native speaker of Brazilian Portuguese, but can assess general pronunciation and audio quality." ] } }, { "id": "f03-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f03-ar-A.wav", "model": "s2.1-pro", "duration": 22.7758 }, "b": { "src": "audio/f03-ar-B.wav", "model": "s2-pro", "duration": 20.686 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples struggle with text fidelity and pronunciation. Sample A garbles the time and adds a syllable, while Sample B mispronounces 'ma'an' significantly. Both have similar audio quality issues.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples struggle with text fidelity and pronunciation. Sample A garbles the time and adds a syllable, while Sample B mispronounces 'ma'an' significantly. Both have similar audio quality issues." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples struggle significantly with Arabic pronunciation, number reading, and phrasing. Sample A produces gibberish for 'معًا', while Sample B misreads the time and adds syllables. Neither is usable." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "2.5/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Garbled pronunciation of the time '7:45'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "text_addition", "evidence": "Added a syllable/word before 'an-nafidha', sounding like 'qurbahum'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "text_substitution", "evidence": "The time '7:45' is incorrectly read as 7:54.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "text_addition", "evidence": "Added an incorrect suffix to 'قرب', sounding like 'قربهم'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'مقعدين' is mispronounced.", "severity": "major", "timestamp_seconds": 3.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'ma'an' at the end of the sentence, sounding like 'panama'.", "severity": "major", "timestamp_seconds": 12.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is disjointed and whispery.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The word 'معًا' is mispronounced as unintelligible gibberish.", "severity": "major", "timestamp_seconds": 12.0 }, { "order": "BA", "category": "pronunciation", "evidence": "The time '7:45' is read with incorrect grammar and phrasing.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.0 } ] }, "limitations": [ "Both samples exhibit noticeable background hiss and unnatural pacing.", "Both samples exhibit severe Arabic pronunciation and prosody errors, making fine-grained dialectal assessment difficult." ] } } ] }, { "id": "f04", "name": "Jen", "slug": "jen-c538c2", "gender": "Female", "rank": 4, "portrait": "portraits/f04.webp", "language": "8 languages", "popularity": { "callers": 154, "calls": 237 }, "pairs": [ { "id": "f04-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-zh-CN-A.wav", "model": "s2-pro", "duration": 15.4848 }, "b": { "src": "audio/f04-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.5761 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a more natural cadence and better captures the requested warm and teasing tones. Sample B sounds slightly more robotic in its phrasing.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a more natural cadence and better captures the requested warm and teasing tones. Sample B sounds slightly more robotic in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly more natural phrasing and pacing compared to Sample A, which sounds more robotic and disjointed. Both samples struggle to fully capture the requested emotional shifts, but B is more pleasant to listen to." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and does not fully capture the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "There are minor synthetic artifacts in the phrasing and cadence.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stilted rhythm throughout the utterance.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and cadence sound unnatural and disjointed throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and does not convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f04-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.943 }, "b": { "src": "audio/f04-ja-JP-B.wav", "model": "s2-pro", "duration": 16.3903 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a much more natural and appropriate delivery. Sample A suffers from excessive breathiness, making the speaker sound distressed rather than warm and reassuring.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a much more natural and appropriate delivery. Sample A suffers from excessive breathiness, making the speaker sound distressed rather than warm and reassuring." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and appropriate delivery for the warm and reassuring prompt, whereas Sample B includes exaggerated breathing artifacts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker sounds out of breath or distressed, which contradicts the 'warm, reassuring' direction.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural, heavy breathing artifacts throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Exaggerated breathing at the start.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [] }, "limitations": [ "I am not a native Japanese speaker, but I can assess general prosody and text fidelity." ] } }, { "id": "f04-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-ko-KR-A.wav", "model": "s2-pro", "duration": 17.2959 }, "b": { "src": "audio/f04-ko-KR-B.wav", "model": "s2.1-pro", "duration": 17.3424 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly more natural pacing and expressiveness, capturing the warm tone better. Sample A sounds a bit more robotic in its cadence.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly more natural pacing and expressiveness, capturing the warm tone better. Sample A sounds a bit more robotic in its cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear intelligibility but share a slightly robotic cadence. Neither stands out significantly in expressiveness or naturalness, making it a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly stiff and unnatural.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Similar to A, the phrasing and cadence have a slightly synthetic, unnatural rhythm.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "other", "evidence": "There is a noticeable reverb or echo effect on the voice.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and pauses feel slightly mechanical throughout the recording.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, but I can evaluate general pronunciation, prosody, and audio quality.", "I am not a native Korean speaker, so subtle nuances in regional accents or advanced prosody may not be fully captured." ] } }, { "id": "f04-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-es-ES-A.wav", "model": "s2.1-pro", "duration": 16.251 }, "b": { "src": "audio/f04-es-ES-B.wav", "model": "s2-pro", "duration": 16.4832 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred because it attempts the requested expressiveness, whereas Sample B is very flat and has a thicker, more disruptive non-native accent.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because it attempts the requested expressiveness, whereas Sample B is very flat and has a thicker, more disruptive non-native accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A successfully captures the warm and reassuring tone requested. Sample B sounds distressed or crying, completely missing the intended delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "other", "evidence": "Unnatural breathy laugh at the very beginning.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speaker sounds distressed or crying, contradicting the 'warm, reassuring' instruction.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Very thick English accent affecting Spanish pronunciation.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and ignores the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature strong non-native (English) accents speaking Spanish." ] } }, { "id": "f04-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-fr-FR-A.wav", "model": "s2-pro", "duration": 14.2309 }, "b": { "src": "audio/f04-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.347 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural delivery and better audio quality. Sample A has noticeable background noise and a slightly robotic cadence.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural delivery and better audio quality. Sample A has noticeable background noise and a slightly robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with a similar non-native accent. They are equally intelligible and follow the text accurately, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background noise and slight distortion throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is somewhat stiff and lacks natural flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the sample.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the sample.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a noticeable non-native accent, which affects naturalness but not strict intelligibility." ] } }, { "id": "f04-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-de-DE-A.wav", "model": "s2-pro", "duration": 15.9956 }, "b": { "src": "audio/f04-de-DE-B.wav", "model": "s2.1-pro", "duration": 16.8083 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural phrasing and intonation than Sample B, which sounds more robotic and disjointed. Both samples mispronounce 'Keks' using an English pronunciation.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly more natural phrasing and intonation than Sample B, which sounds more robotic and disjointed. Both samples mispronounce 'Keks' using an English pronunciation." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural delivery with a less pronounced foreign accent compared to Sample A. Both samples accurately convey the text and handle the time format well." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Slight non-native accent, though less intrusive than in Sample A.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are stiff and unnatural throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign (American) accent on words like 'reserviert' and 'Keks'.", "severity": "minor", "timestamp_seconds": 5.0 } ] }, "limitations": [ "Both samples exhibit a slight non-native accent, but it does not severely impact intelligibility." ] } }, { "id": "f04-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-pt-BR-A.wav", "model": "s2-pro", "duration": 17.6907 }, "b": { "src": "audio/f04-pt-BR-B.wav", "model": "s2.1-pro", "duration": 16.1814 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples fail to provide a convincing Brazilian Portuguese accent, sounding distinctly European. Sample A is preferred for having slightly more natural phrasing and cadence than Sample B.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples fail to provide a convincing Brazilian Portuguese accent, sounding distinctly European. Sample A is preferred for having slightly more natural phrasing and cadence than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, instead speaking with a European Portuguese accent. Sample B is preferred for its more natural pacing and better emotional delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The accent sounds like European Portuguese rather than the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "The accent sounds like European Portuguese rather than the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is somewhat stiff and unnatural.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat choppy and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese.", "Both samples exhibit a European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "f04-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f04-ar-A.wav", "model": "s2-pro", "duration": 18.5498 }, "b": { "src": "audio/f04-ar-B.wav", "model": "s2.1-pro", "duration": 18.7123 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample A is slightly more robotic in its pacing.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample A is slightly more robotic in its pacing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit significant robotic artifacts and unnatural, choppy phrasing. Neither successfully conveys the requested warm or teasing delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing feels slightly mechanical and lacks dynamic range.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is unnatural with awkward pauses disrupting the flow.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is flat and does not match the requested emotional cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The speech is highly disjointed with unnatural pauses between words.", "severity": "major", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or gently teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, so my evaluation of subtle accent variations is limited.", "I am not a native speaker of Modern Standard Arabic, but the robotic prosody and unnatural pauses are clearly audible." ] } } ] }, { "id": "f05", "name": "vanessa", "slug": "vanessa-4f1786", "gender": "Female", "rank": 5, "portrait": "portraits/f05.webp", "language": "8 languages", "popularity": { "callers": 99, "calls": 123 }, "pairs": [ { "id": "f05-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-zh-CN-A.wav", "model": "s2.1-pro", "duration": 15.9956 }, "b": { "src": "audio/f05-zh-CN-B.wav", "model": "s2-pro", "duration": 18.8052 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more natural and expressive delivery, effectively capturing the requested warm and teasing tones. Sample B sounds slightly more robotic and less emotionally varied.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery, effectively capturing the requested warm and teasing tones. Sample B sounds slightly more robotic and less emotionally varied." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational pace and phrasing. Sample A is overly slow and exhibits robotic prosody, making it sound less humanlike." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The tone is somewhat flat and does not fully capture the requested warmth or teasing.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is unnaturally slow and lacks conversational flow.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native speaker of all regional Mandarin accents." ] } }, { "id": "f05-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-ja-JP-A.wav", "model": "s2-pro", "duration": 23.0312 }, "b": { "src": "audio/f05-ja-JP-B.wav", "model": "s2.1-pro", "duration": 23.8207 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit highly robotic prosody and fail to convey the requested warm and teasing emotions. They are equally unnatural and disjointed.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit highly robotic prosody and fail to convey the requested warm and teasing emotions. They are equally unnatural and disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text with clear intelligibility. However, both suffer from highly unnatural, robotic prosody and flat intonation that fails to capture the requested emotional shifts." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech is disjointed with unnatural pauses and flat intonation throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to sound warm, reassuring, or gently teasing as requested.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The speech exhibits flat, mechanical intonation throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is very unnatural, with disjointed phrasing and flat pitch.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall intonation and phrasing sound mechanical and lack natural Japanese cadence.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit significant robotic prosody and unnatural intonation for Japanese." ] } }, { "id": "f05-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-ko-KR-A.wav", "model": "s2.1-pro", "duration": 20.8253 }, "b": { "src": "audio/f05-ko-KR-B.wav", "model": "s2-pro", "duration": 21.7542 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural pacing and a better realization of the requested emotional shift compared to Sample B, which feels a bit more subdued and slow.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly more natural pacing and a better realization of the requested emotional shift compared to Sample B, which feels a bit more subdued and slow." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B has slightly better pacing and a more natural conversational tone, whereas Sample A sounds a bit slow and robotic." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat slow and unnatural.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle emotional shifts in Korean may be limited by non-native nuances.", "I am not a native speaker of Korean, so subtle prosodic nuances may be missed." ] } }, { "id": "f05-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-es-ES-A.wav", "model": "s2-pro", "duration": 19.0838 }, "b": { "src": "audio/f05-es-ES-B.wav", "model": "s2.1-pro", "duration": 20.1752 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural cadence and better intelligibility. Sample A has a somewhat robotic prosody and a slight accent that sounds less native to the requested locale.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural cadence and better intelligibility. Sample A has a somewhat robotic prosody and a slight accent that sounds less native to the requested locale." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a severe English accent when reading the Spanish text, resulting in unnatural pronunciation and prosody. Neither successfully captures the requested locale or delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation feel slightly mechanical and disjointed.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Slight non-native accent on some vowels and consonants.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text, such as in 'reservado' and 'galleta'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not very pronounced.", "severity": "minor", "timestamp_seconds": 15.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text, such as in 'reservado' and 'galleta'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stilted and unnatural cadence.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Spanish (Spain) and may miss subtle regional accent nuances." ] } }, { "id": "f05-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-fr-FR-A.wav", "model": "s2-pro", "duration": 19.0606 }, "b": { "src": "audio/f05-fr-FR-B.wav", "model": "s2.1-pro", "duration": 16.6922 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B features native-sounding French pronunciation and appropriate expressiveness. Sample A has a severe English accent, mispronouncing almost all French vowels and nasal sounds, making it sound highly unnatural.", "confidence": 0.875, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.95, "rationale": "Sample B features native-sounding French pronunciation and appropriate expressiveness. Sample A has a severe English accent, mispronouncing almost all French vowels and nasal sounds, making it sound highly unnatural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a heavy American accent that distorts French pronunciation. Sample A is preferred because its prosody and flow are slightly more natural, whereas Sample B is very choppy." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, mispronouncing words like 'réservé' as 'rezervi' and 'journée' as 'joorni'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy American accent throughout, e.g., 'fenêtre' and 'journée'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Choppy and unnatural pacing between words.", "severity": "major", "timestamp_seconds": 7.0 } ], "B": [ { "order": "BA", "category": "pronunciation", "evidence": "Heavy American accent throughout, e.g., 'Prends ton temps' and 'deux'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a very strong foreign (American) accent, significantly impacting naturalness for the requested French locale." ] } }, { "id": "f05-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-de-DE-A.wav", "model": "s2.1-pro", "duration": 20.1055 }, "b": { "src": "audio/f05-de-DE-B.wav", "model": "s2-pro", "duration": 20.8486 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a strong foreign accent that impacts naturalness and pronunciation, such as pronouncing 'Keks' like the English word 'cakes'. Neither sample is clearly superior.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a strong foreign accent that impacts naturalness and pronunciation, such as pronouncing 'Keks' like the English word 'cakes'. Neither sample is clearly superior." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural pacing and better expressiveness. Both samples share a pronunciation error on the word 'Keks', but B's overall delivery is smoother." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Keks' pronounced as 'cakes'.", "severity": "major", "timestamp_seconds": 18.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 18.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, e.g., 'Keks' pronounced as 'cakes'.", "severity": "major", "timestamp_seconds": 18.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "major", "timestamp_seconds": 18.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat stilted and slow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong foreign (American) accent, affecting the pronunciation of several German words.", "Both samples mispronounce 'Keks' as the English word 'cakes'." ] } }, { "id": "f05-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-pt-BR-A.wav", "model": "s2.1-pro", "duration": 20.3842 }, "b": { "src": "audio/f05-pt-BR-B.wav", "model": "s2-pro", "duration": 22.0792 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples exhibit a strong foreign accent that affects naturalness. Sample B is preferred for having slightly smoother prosody and better flow compared to Sample A's disjointed delivery.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a strong foreign accent that affects naturalness. Sample B is preferred for having slightly smoother prosody and better flow compared to Sample A's disjointed delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a heavy foreign accent that detracts from the naturalness of the Brazilian Portuguese text. Sample A is slightly preferred for its better pacing and slightly warmer tone, whereas Sample B feels more rushed and robotic." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent throughout, affecting vowels and nasal sounds like 'Não' and 'amanhã'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is disjointed with unnatural pauses and cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat rushed and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable foreign accent affecting words like 'amanhã' and 'janela'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, affecting vowels and consonants (e.g., 'Não', 'pressa').", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Non-native speaker evaluating Brazilian Portuguese, though the foreign accent in both samples is highly prominent.", "Both samples feature a very strong non-native (likely English) accent, which impacts naturalness and intelligibility in Brazilian Portuguese." ] } }, { "id": "f05-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f05-ar-A.wav", "model": "s2-pro", "duration": 22.7294 }, "b": { "src": "audio/f05-ar-B.wav", "model": "s2.1-pro", "duration": 21.1272 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to slightly better phrasing and naturalness. Sample B suffers from abrupt, unnatural pauses that disrupt the flow of the sentence.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to slightly better phrasing and naturalness. Sample B suffers from abrupt, unnatural pauses that disrupt the flow of the sentence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from robotic prosody and unnatural pauses. Sample B is slightly preferred due to marginally better pacing and fewer choppy transitions compared to Sample A." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural phrasing and awkward pauses, though slightly less severe than A.", "severity": "major", "timestamp_seconds": 8.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural, abrupt pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The tone remains flat throughout, ignoring the performance cues.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly unnatural pacing and disjointed phrasing throughout, such as the awkward pause before 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 } ] }, "limitations": [ "Both samples exhibit robotic prosody and fail to fully capture the requested emotional shifts.", "Both samples exhibit highly unnatural, robotic prosody with awkward pauses, making it difficult to assess subtle expressiveness." ] } } ] }, { "id": "f06", "name": "Alicia a young white women with a curvy body", "slug": "alicia-a-young-white-women-with-a-curvy-body-b4f9fb", "gender": "Female", "rank": 6, "portrait": "portraits/f06.webp", "language": "8 languages", "popularity": { "callers": 94, "calls": 131 }, "pairs": [ { "id": "f06-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-zh-CN-A.wav", "model": "s2-pro", "duration": 14.8578 }, "b": { "src": "audio/f06-zh-CN-B.wav", "model": "s2.1-pro", "duration": 15.4383 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers more natural prosody and better captures the warm, gently teasing delivery requested. Sample B sounds slightly more stiff and robotic in its phrasing.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers more natural prosody and better captures the warm, gently teasing delivery requested. Sample B sounds slightly more stiff and robotic in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a heavy non-native accent that impacts tonal accuracy. Sample A is preferred for having slightly more natural phrasing and better overall flow, whereas Sample B sounds more stilted." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent impacts Mandarin tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat stilted and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff phrasing throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects tonal accuracy throughout the sample.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, which affects tonal accuracy and naturalness." ] } }, { "id": "f06-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-ja-JP-A.wav", "model": "s2-pro", "duration": 16.6457 }, "b": { "src": "audio/f06-ja-JP-B.wav", "model": "s2.1-pro", "duration": 17.6674 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples successfully deliver the target text but suffer from a very strong non-native accent that impacts naturalness. There is no clear winner as both share similar prosodic limitations.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples successfully deliver the target text but suffer from a very strong non-native accent that impacts naturalness. There is no clear winner as both share similar prosodic limitations." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from unnatural, robotic prosody with incorrect pitch accents. Sample B is preferred because Sample A substitutes a word, altering the grammar slightly." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pitch accents and disjointed phrasing throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_substitution", "evidence": "Heard 'suwatta' instead of expected 'suwatte'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pitch accents and disjointed phrasing throughout.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody.", "Both samples exhibit non-native pitch accents and robotic prosody." ] } }, { "id": "f06-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-ko-KR-A.wav", "model": "s2.1-pro", "duration": 16.7619 }, "b": { "src": "audio/f06-ko-KR-B.wav", "model": "s2-pro", "duration": 17.3191 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly better flow and prosody than Sample A, which is noticeably choppy. Both samples suffer from a strong foreign accent.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly better flow and prosody than Sample A, which is noticeably choppy. Both samples suffer from a strong foreign accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers more natural prosody and clearer pronunciation, effectively capturing the intended warmth and teasing tone. Sample B sounds slightly robotic and less expressive." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent; '내일' sounds like '내야'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is choppy with unnatural pauses between words.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and phrasing throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent; '우리 둘이' sounds like '우리 추리'.", "severity": "major", "timestamp_seconds": 8.5 } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f06-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-es-ES-A.wav", "model": "s2.1-pro", "duration": 15.2061 }, "b": { "src": "audio/f06-es-ES-B.wav", "model": "s2-pro", "duration": 14.9507 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe pronunciation errors, most notably pronouncing the silent 'h' in Spanish words like 'He' and 'hablar'. They are equally flawed in their non-native delivery.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe pronunciation errors, most notably pronouncing the silent 'h' in Spanish words like 'He' and 'hablar'. They are equally flawed in their non-native delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe English accent that heavily impacts naturalness and pronunciation. Sample A is slightly preferred due to marginally better expressiveness and prosody." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He' and 'hablar' with an English 'h' sound.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'He reservado' pronounced with an English 'h' and 'r'.", "severity": "major", "timestamp_seconds": 0.01 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He' and 'hablar' with an English 'h' sound.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'He reservado' pronounced with an English 'h' and 'r'.", "severity": "major", "timestamp_seconds": 0.01 } ] }, "limitations": [ "Both samples exhibit a very strong non-native (English) accent.", "Both samples exhibit a severe non-native (English) accent, making it difficult to assess native Spanish prosody." ] } }, { "id": "f06-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-fr-FR-A.wav", "model": "s2-pro", "duration": 13.8361 }, "b": { "src": "audio/f06-fr-FR-B.wav", "model": "s2.1-pro", "duration": 14.0219 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from severe pronunciation issues due to a heavy English accent. Sample A is slightly preferred because it lacks the audio choppiness present in Sample B.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from severe pronunciation issues due to a heavy English accent. Sample A is slightly preferred because it lacks the audio choppiness present in Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples exhibit a strong non-native accent that impacts naturalness. Sample B has slightly better audio quality and smoother phrasing." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, distorting vowels and consonants (e.g., 'Prends ton temps').", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, distorting vowels and consonants.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "click_or_glitch", "evidence": "Slight choppiness in the audio.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, e.g., 'Prends ton temps'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing feels disjointed and unnatural.", "severity": "minor", "timestamp_seconds": 5.0 } ] }, "limitations": [ "Both samples exhibit a severe non-native (English) accent that heavily distorts French pronunciation.", "Both samples feature a strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f06-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-de-DE-A.wav", "model": "s2.1-pro", "duration": 16.0885 }, "b": { "src": "audio/f06-de-DE-B.wav", "model": "s2-pro", "duration": 16.0652 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a noticeable non-native accent. Sample B is preferred for slightly better pacing and clearer articulation, particularly at the beginning.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a noticeable non-native accent. Sample B is preferred for slightly better pacing and clearer articulation, particularly at the beginning." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a very strong English accent that impacts the pronunciation of German words. Neither successfully captures the requested delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'reden' and 'genießen'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, pronouncing 'Keks' as 'cakes'.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent on 'reserviert' and 'reden'.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, pronouncing 'Keks' as 'cakes'.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f06-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-pt-BR-A.wav", "model": "s2-pro", "duration": 16.042 }, "b": { "src": "audio/f06-pt-BR-B.wav", "model": "s2.1-pro", "duration": 16.0188 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe English accent, mispronouncing basic Portuguese phonemes. Sample A is slightly more expressive and has marginally better phrasing than B, which sounds flatter.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a severe English accent, mispronouncing basic Portuguese phonemes. Sample A is slightly more expressive and has marginally better phrasing than B, which sounds flatter." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples read the text accurately but suffer from a very strong foreign accent that reduces naturalness. Neither clearly outperforms the other." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'janela' and 'juntos' with an English 'j' sound.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting words like 'janela' and 'biscoito'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'janela' and 'juntos' with an English 'j' sound.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting words like 'precisa' and 'lugares'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong non-native (English) accent, making it difficult to assess native-level prosody.", "Both samples feature a very strong non-native (likely English) accent that affects the naturalness of the Brazilian Portuguese." ] } }, { "id": "f06-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f06-ar-A.wav", "model": "s2.1-pro", "duration": 16.9476 }, "b": { "src": "audio/f06-ar-B.wav", "model": "s2-pro", "duration": 17.3424 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B correctly reads the time (7:45), whereas Sample A omits the minutes. Sample B also has slightly better prosody and audio quality.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B correctly reads the time (7:45), whereas Sample A omits the minutes. Sample B also has slightly better prosody and audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a more accurate reading of the text, particularly the time, and has clearer pronunciation. Sample B omits part of the time and has a slight pronunciation error early on." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "2.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_omission", "evidence": "Omitted the minutes '45' when reading the time '7:45'.", "severity": "major", "timestamp_seconds": 3.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounced 'للاستعجال' incorrectly, missing the 'j' sound.", "severity": "minor", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "text_omission", "evidence": "Omitted the '5' in '7:45', saying 'sab'a wa arba'in'.", "severity": "major", "timestamp_seconds": 3.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural phrasing in some segments.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of Arabic number normalization is based on standard conventions.", "I am not a native speaker of Arabic, but can identify clear pronunciation and text fidelity errors." ] } } ] }, { "id": "f07", "name": "Hannah", "slug": "hannah-69dda7", "gender": "Female", "rank": 7, "portrait": "portraits/f07.webp", "language": "8 languages", "popularity": { "callers": 87, "calls": 108 }, "pairs": [ { "id": "f07-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-zh-CN-A.wav", "model": "s2-pro", "duration": 17.9461 }, "b": { "src": "audio/f07-zh-CN-B.wav", "model": "s2.1-pro", "duration": 17.5281 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A sounds much more natural with a native-like Mandarin accent and appropriate prosody. Sample B has a noticeable foreign accent and some tonal inaccuracies that affect naturalness.", "confidence": 0.925, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A sounds much more natural with a native-like Mandarin accent and appropriate prosody. Sample B has a noticeable foreign accent and some tonal inaccuracies that affect naturalness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.95, "rationale": "Sample B provides native-like pronunciation and natural prosody that perfectly matches the requested warm delivery. Sample A suffers from a severe foreign accent and incorrect tones, significantly degrading naturalness and intelligibility." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable foreign accent and some tonal inaccuracies throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent with incorrect tones throughout, e.g., '不着急' and '45分'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural cadence and phrasing due to the heavy accent.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f07-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-ja-JP-A.wav", "model": "s2-pro", "duration": 19.9894 }, "b": { "src": "audio/f07-ja-JP-B.wav", "model": "s2.1-pro", "duration": 20.1288 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a strong non-native accent with incorrect pitch accents throughout the recording. They are equally intelligible and follow the text accurately, resulting in a tie.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent with incorrect pitch accents throughout the recording. They are equally intelligible and follow the text accurately, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides a more natural delivery and correctly pronounces the text. Sample A mispronounces '明日' as 'asa' instead of 'ashita', creating a contradictory phrase." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speaker has a noticeable non-native accent with incorrect pitch accents on many words.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speaker has a noticeable non-native accent with incorrect pitch accents on many words.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "other", "evidence": "There is an unnatural chuckle or breathy artifact at the very beginning.", "severity": "minor", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日の夜' as 'asa no yoru' instead of 'ashita no yoru'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Evaluation of pitch accent correctness is based on standard Tokyo Japanese." ] } }, { "id": "f07-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-ko-KR-A.wav", "model": "s2-pro", "duration": 20.2448 }, "b": { "src": "audio/f07-ko-KR-B.wav", "model": "s2.1-pro", "duration": 18.5266 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a more expressive and natural delivery that better captures the requested warm and teasing tones. Sample B is somewhat flat and monotonous in comparison.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more expressive and natural delivery that better captures the requested warm and teasing tones. Sample B is somewhat flat and monotonous in comparison." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides clearer audio quality and more natural prosody. Sample A sounds slightly muffled and exhibits a more robotic cadence." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and lacks the requested warm and teasing shifts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing and cadence throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Audio sounds slightly muffled with minor artifacts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle accent or prosody nuances may not be fully evaluated." ] } }, { "id": "f07-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-es-ES-A.wav", "model": "s2.1-pro", "duration": 18.1318 }, "b": { "src": "audio/f07-es-ES-B.wav", "model": "s2-pro", "duration": 19.1071 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a heavy foreign accent that severely impacts naturalness. Sample A is preferred slightly for having a marginally better flow and less robotic pacing than Sample B.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a heavy foreign accent that severely impacts naturalness. Sample A is preferred slightly for having a marginally better flow and less robotic pacing than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent, mispronouncing key Spanish phonemes (like the 'ñ' in 'mañana'). Neither sounds like a native Spanish speaker, resulting in a tie for poor locale accuracy." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the Spanish text, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'mañana' pronounced without the 'ñ' sound.", "severity": "major", "timestamp_seconds": 4.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is slow and disjointed.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'mañana' pronounced without the 'ñ' sound.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "text_omission", "evidence": "Omitted 'y' in the time, saying 'siete cuarenta y cinco' instead of 'siete y cuarenta y cinco'.", "severity": "minor", "timestamp_seconds": 5.0 } ] }, "limitations": [ "Both samples exhibit a severe English accent when speaking Spanish.", "Both samples exhibit a severe non-native accent, making it difficult to assess natural Spanish prosody." ] } }, { "id": "f07-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-fr-FR-A.wav", "model": "s2.1-pro", "duration": 16.8547 }, "b": { "src": "audio/f07-fr-FR-B.wav", "model": "s2-pro", "duration": 17.1798 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly clearer pronunciation and better intelligibility. Sample A has minor pronunciation issues at the beginning.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly clearer pronunciation and better intelligibility. Sample A has minor pronunciation issues at the beginning." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from a severe non-native accent that impacts intelligibility. Sample B is slightly clearer and easier to understand than Sample A." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronunciation of 'Prends ton temps' is slightly muffled and inaccurate.", "severity": "minor", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, though slightly clearer than A.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is extremely whispered rather than just warm and reassuring.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' and 'réservé'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples use a heavily whispered delivery which may affect intelligibility.", "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f07-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-de-DE-A.wav", "model": "s2-pro", "duration": 20.3842 }, "b": { "src": "audio/f07-de-DE-B.wav", "model": "s2.1-pro", "duration": 18.3873 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a strong foreign accent and an unnaturally breathy delivery. They are equally intelligible and preserve the text accurately, resulting in a tie.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a strong foreign accent and an unnaturally breathy delivery. They are equally intelligible and preserve the text accurately, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a severe foreign accent that degrades naturalness and intelligibility. Sample B is slightly preferred due to marginally cleaner audio quality." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, e.g., 'Lass dir Zeit'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnaturally breathy and slow delivery.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout the recording, affecting words like 'Lass', 'Zeit', and 'Keks'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnaturally breathy delivery.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout the recording, affecting words like 'Lass', 'Zeit', and 'Keks'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody.", "Both samples feature a severe English accent, making them sound like a non-native speaker reading German." ] } }, { "id": "f07-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-pt-BR-A.wav", "model": "s2.1-pro", "duration": 19.7572 }, "b": { "src": "audio/f07-pt-BR-B.wav", "model": "s2-pro", "duration": 19.8501 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B has better audio quality and naturalness, though it uses a European Portuguese accent instead of the requested Brazilian. Sample A attempts a Brazilian accent but has a strong foreign accent and mispronunciations.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B has better audio quality and naturalness, though it uses a European Portuguese accent instead of the requested Brazilian. Sample A attempts a Brazilian accent but has a strong foreign accent and mispronunciations." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy foreign accent and robotic prosody, making them sound unnatural for Brazilian Portuguese. They both read the text accurately, but neither provides a convincing or expressive performance." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'janela' with an English 'j' sound.", "severity": "major", "timestamp_seconds": 9.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Pronounces 'juntos' as 'juntas'.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting words like 'Reservei' and 'juntos'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is slow and lacks natural intonation and rhythm.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Uses a European Portuguese accent (pt-PT) instead of the requested Brazilian Portuguese (pt-BR).", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting words like 'Reservei' and 'lugares'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is slow, deliberate, and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Portuguese, but the distinction between Brazilian and European Portuguese accents is apparent.", "Both samples feature a strong non-native accent, which affects naturalness and intelligibility." ] } }, { "id": "f07-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f07-ar-A.wav", "model": "s2-pro", "duration": 22.683 }, "b": { "src": "audio/f07-ar-B.wav", "model": "s2.1-pro", "duration": 22.5901 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to slightly better intelligibility and fewer word-level mispronunciations. Both samples suffer from a strong foreign accent, but Sample B has more noticeable phonetic errors.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to slightly better intelligibility and fewer word-level mispronunciations. Both samples suffer from a strong foreign accent, but Sample B has more noticeable phonetic errors." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better naturalness and pacing compared to Sample A, which sounds a bit more robotic in its delivery." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout affects overall intelligibility.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Reads '7:45' as 'sab'a khamsa wa arba'in', omitting the conjunction.", "severity": "minor", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The transition to gentle teasing is somewhat flat.", "severity": "minor", "timestamp_seconds": 16.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'qurb' (near) as something like 'qurdammi'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'linatruk' (let's leave) as 'lanantruk'.", "severity": "major", "timestamp_seconds": 17.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing feels slightly unnatural and mechanical in the middle section.", "severity": "minor", "timestamp_seconds": 8.5 } ] }, "limitations": [ "Both samples feature a heavy non-native accent that affects the naturalness and intelligibility of the Arabic.", "I am not a native speaker of Modern Standard Arabic, so subtle pronunciation nuances might be missed." ] } } ] }, { "id": "f08", "name": "Melissa \"Miel\" De Luna", "slug": "melissa-miel-de-luna-b1d93b", "gender": "Female", "rank": 8, "portrait": "portraits/f08.webp", "language": "8 languages", "popularity": { "callers": 83, "calls": 103 }, "pairs": [ { "id": "f08-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-zh-CN-A.wav", "model": "s2.1-pro", "duration": 20.477 }, "b": { "src": "audio/f08-zh-CN-B.wav", "model": "s2-pro", "duration": 20.8253 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a more natural and expressive delivery that better captures the intended warm and reassuring tone. Sample A sounds slightly more robotic and has minor audio quality issues.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery that better captures the intended warm and reassuring tone. Sample A sounds slightly more robotic and has minor audio quality issues." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better captures the requested warm and reassuring tone. Sample B sounds slightly more robotic and stiff in its prosody." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The phrasing and cadence sound somewhat unnatural and disjointed.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Slight background noise or artifacting is audible.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall cadence and phrasing sound somewhat stiff and synthetic compared to natural speech.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [ "I am not a native speaker of Mandarin Chinese, so my evaluation of subtle accent or prosody nuances may be limited." ] } }, { "id": "f08-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-ja-JP-A.wav", "model": "s2-pro", "duration": 23.7046 }, "b": { "src": "audio/f08-ja-JP-B.wav", "model": "s2.1-pro", "duration": 23.4956 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more stable and natural delivery with appropriate pacing and intonation. Sample B exhibits a slightly shaky voice quality and minor unnatural intonation patterns.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more stable and natural delivery with appropriate pacing and intonation. Sample B exhibits a slightly shaky voice quality and minor unnatural intonation patterns." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a slightly more natural pacing and avoids the overly breathy, exaggerated delivery found in Sample B, making it a better fit for the requested conversational tone." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is overly breathy and slow, sounding somewhat unnatural for the context.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural intonation on '予約しておいたんだ' and '話そう'.", "severity": "minor", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly synthetic cadence in some phrases.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional accents is limited.", "Evaluation of subtle Japanese pitch accent variations is limited." ] } }, { "id": "f08-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-ko-KR-A.wav", "model": "s2-pro", "duration": 22.2185 }, "b": { "src": "audio/f08-ko-KR-B.wav", "model": "s2.1-pro", "duration": 24.2619 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers slightly better pacing and naturalness than Sample A, which is overly slow and disjointed. Both samples suffer from noticeable audio artifacts and hiss.", "confidence": 0.7, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly better pacing and naturalness than Sample A, which is overly slow and disjointed. Both samples suffer from noticeable audio artifacts and hiss." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers a slightly more natural delivery, though it is overly breathy. Sample A suffers from disjointed pacing and robotic prosody." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Overly slow and disjointed pacing throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and audio artifacts.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and whispered, which is a bit extreme for the requested delivery.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and audio artifacts.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Some unnatural cadence and phrasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is disjointed and unnatural throughout the sample.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Non-native speaker; evaluating based on general phonetic and prosodic principles." ] } }, { "id": "f08-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-es-ES-A.wav", "model": "s2-pro", "duration": 20.2913 }, "b": { "src": "audio/f08-es-ES-B.wav", "model": "s2.1-pro", "duration": 21.7542 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from heavy non-native accents that detract from naturalness. Sample B is preferred slightly due to marginally better audio quality and less background hiss.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from heavy non-native accents that detract from naturalness. Sample B is preferred slightly due to marginally better audio quality and less background hiss." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a strong foreign accent that impacts naturalness. Sample B is slightly preferred due to better audio clarity and pacing." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'He reservado' pronounced with an English 'H' sound.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent affecting vowel quality and consonant articulation.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly robotic and flat delivery.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit strong non-native accents, making it difficult to assess native-level prosody.", "Both samples exhibit a strong non-native (English) accent, which affects naturalness." ] } }, { "id": "f08-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-fr-FR-A.wav", "model": "s2-pro", "duration": 20.8021 }, "b": { "src": "audio/f08-fr-FR-B.wav", "model": "s2.1-pro", "duration": 18.8517 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a very strong foreign accent that heavily distorts French pronunciation. Sample A is slightly preferred due to a more stable vocal quality.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples feature a very strong foreign accent that heavily distorts French pronunciation. Sample A is slightly preferred due to a more stable vocal quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a very heavy non-native accent with significant mispronunciations, making them equally unnatural for a French locale." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent distorts 'Prends ton temps' into 'Prong ton ton'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'Prends ton temps' with hard consonants.", "severity": "major", "timestamp_seconds": 0.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy foreign accent distorts 'Prends ton temps' into 'Prong ton ton'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronunciation of 'Prends ton temps' with hard consonants.", "severity": "major", "timestamp_seconds": 0.5 } ] }, "limitations": [ "Both samples feature a heavy non-native accent that affects the pronunciation of several words." ] } }, { "id": "f08-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-de-DE-A.wav", "model": "s2.1-pro", "duration": 23.5421 }, "b": { "src": "audio/f08-de-DE-B.wav", "model": "s2-pro", "duration": 22.7062 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit highly robotic prosody and unnatural delivery. Sample A has better pronunciation but worse pacing, while Sample B has a strong foreign accent. Neither successfully conveys the requested emotions.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit highly robotic prosody and unnatural delivery. Sample A has better pronunciation but worse pacing, while Sample B has a strong foreign accent. Neither successfully conveys the requested emotions." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and appropriate delivery. Sample B is overly exaggerated and cartoonish, failing to match the requested warm and gently teasing tone." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are highly unnatural and disjointed throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is flat and robotic, failing to convey the requested warm or teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is highly exaggerated and cartoonish, not warm or gently teasing.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong foreign accent affects the pronunciation of several words, such as 'dir' and 'reserviert'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The speech sounds mechanical with unnatural pauses and cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to capture the warm and teasing emotional cues, sounding flat instead.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is slightly slow and mechanical in places.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native German speaker, but can assess general pronunciation and prosody." ] } }, { "id": "f08-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-pt-BR-A.wav", "model": "s2.1-pro", "duration": 23.6118 }, "b": { "src": "audio/f08-pt-BR-B.wav", "model": "s2-pro", "duration": 22.3811 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A uses the requested Brazilian Portuguese locale, whereas Sample B uses a distinct European Portuguese accent. Despite Sample A's robotic prosody, it adheres better to the target locale.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A uses the requested Brazilian Portuguese locale, whereas Sample B uses a distinct European Portuguese accent. Despite Sample A's robotic prosody, it adheres better to the target locale." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese accent, sounding distinctly European. Sample B is preferred for having slightly more natural pacing and prosody, whereas Sample A is overly slow and robotic." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation sound slightly mechanical throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "wrong_language", "evidence": "The speaker uses a European Portuguese accent and pronunciation instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is unnaturally slow with disjointed phrasing.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "f08-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f08-ar-A.wav", "model": "s2.1-pro", "duration": 27.4662 }, "b": { "src": "audio/f08-ar-B.wav", "model": "s2-pro", "duration": 23.1473 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from robotic prosody and disjointed pacing. Sample B is slightly preferred for having marginally better flow and intelligibility.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from robotic prosody and disjointed pacing. Sample B is slightly preferred for having marginally better flow and intelligibility." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers more natural phrasing and prosody. Sample B has unnatural pauses and a grammatical error in pronunciation." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Highly disjointed pacing and unnatural pauses throughout the utterance.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Awkward pronunciation of 'قرب النافذة' with incorrect vowel endings.", "severity": "minor", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'طلب' as 'طلبا' which is grammatically incorrect in this context.", "severity": "minor", "timestamp_seconds": 19.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and choppy delivery, particularly in the middle section.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly disjointed phrasing in some sentences.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit significant synthetic artifacts and unnatural phrasing.", "I am not a native speaker of Modern Standard Arabic, but I can identify basic pronunciation and prosody issues." ] } } ] }, { "id": "f09", "name": "red head female", "slug": "red-head-female-f9dab4", "gender": "Female", "rank": 9, "portrait": "portraits/f09.webp", "language": "8 languages", "popularity": { "callers": 81, "calls": 98 }, "pairs": [ { "id": "f09-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-zh-CN-A.wav", "model": "s2-pro", "duration": 18.7588 }, "b": { "src": "audio/f09-zh-CN-B.wav", "model": "s2.1-pro", "duration": 17.3888 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from poor audio quality and muffled sound. Sample A is preferred because its slower pacing and tone better match the requested warm and reassuring delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from poor audio quality and muffled sound. Sample A is preferred because its slower pacing and tone better match the requested warm and reassuring delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from poor audio quality and excessive whispering, but Sample B is slightly louder and more intelligible than Sample A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3.5, "audioQuality": 2 }, "B": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 3, "audioQuality": 2 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "The audio sounds muffled and heavily compressed throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "The recording is muffled and has noticeable background hiss.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than a natural warm tone.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "The audio is muffled and has noticeable compression artifacts.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The pacing is a bit rushed, lacking the requested warm and reassuring tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "The entire recording is extremely muffled and quiet, making it difficult to hear.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than a natural warm tone.", "severity": "minor", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit significant audio quality degradation, sounding muffled with vocoder artifacts.", "Both samples are extremely quiet and whispered, making detailed phonetic analysis difficult." ] } }, { "id": "f09-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-ja-JP-A.wav", "model": "s2-pro", "duration": 20.6628 }, "b": { "src": "audio/f09-ja-JP-B.wav", "model": "s2.1-pro", "duration": 20.895 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred because it is more intelligible. Sample B suffers from severe slurring and garbling, particularly during the time phrase, making it difficult to understand. Both samples have poor, muffled audio quality.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because it is more intelligible. Sample B suffers from severe slurring and garbling, particularly during the time phrase, making it difficult to understand. Both samples have poor, muffled audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B has noticeably better audio clarity and intelligibility. Sample A suffers from severe muffling and distortion, making the whispered delivery difficult to understand." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 3, "intelligibility": 2, "expressiveness": 3, "audioQuality": 2 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "The audio is muffled and has noticeable background hiss throughout.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and slight breathy distortion throughout the whispered delivery.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "The phrase '7時45分に' is heavily slurred and garbled.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "AB", "category": "distortion", "evidence": "The audio is heavily distorted and muffled throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "The entire recording is heavily muffled with significant audio degradation and hiss.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit poor audio quality, making fine-grained pronunciation assessment difficult.", "Both samples use a heavily whispered, ASMR-like delivery which makes assessing standard prosody difficult." ] } }, { "id": "f09-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-ko-KR-A.wav", "model": "s2-pro", "duration": 20.3842 }, "b": { "src": "audio/f09-ko-KR-B.wav", "model": "s2.1-pro", "duration": 17.9461 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B provides a much more natural and conversational delivery. Sample A is overly whispered and slow, making it sound unnatural and less intelligible, while also suffering from worse audio quality.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a much more natural and conversational delivery. Sample A is overly whispered and slow, making it sound unnatural and less intelligible, while also suffering from worse audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly better intelligibility and a more fitting expressive delivery. Both samples suffer from noticeable background noise and a somewhat muffled audio quality." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 2.5 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is an exaggerated whisper throughout, missing the gentle teasing cue.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and slight distortion on the voice.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss throughout the recording.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "other", "evidence": "Slight background noise present.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The word '내일' sounds like '내어'.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and slightly muffled audio.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit some background noise, but A's is more pronounced.", "I am not a native Korean speaker, so subtle accent nuances may not be fully evaluated." ] } }, { "id": "f09-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-es-ES-A.wav", "model": "s2.1-pro", "duration": 18.0158 }, "b": { "src": "audio/f09-es-ES-B.wav", "model": "s2-pro", "duration": 17.1798 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples use an exaggerated whispered delivery with a noticeable non-native accent. Sample B is preferred slightly due to marginally clearer audio quality with less background hiss.", "confidence": 0.65, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Both samples use an exaggerated whispered delivery with a noticeable non-native accent. Sample B is preferred slightly due to marginally clearer audio quality with less background hiss." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A is preferred for slightly better phrasing of the time and a more consistent accent for the requested locale. Both samples suffer from excessive breathiness and background noise." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is heavily whispered rather than naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Added 'y' when reading the time: 'siete y cuarenta y cinco'.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and excessive breathiness throughout.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is heavily whispered rather than naturally warm.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and excessive breathiness throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a heavy non-native accent.", "Both samples use an exaggerated whispered delivery.", "Both samples exhibit excessive breathiness and background hiss, limiting overall audio quality." ] } }, { "id": "f09-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.1597 }, "b": { "src": "audio/f09-fr-FR-B.wav", "model": "s2-pro", "duration": 16.2975 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a heavy non-native accent and a whispering delivery. Neither clearly outperforms the other in intelligibility or naturalness.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy non-native accent and a whispering delivery. Neither clearly outperforms the other in intelligibility or naturalness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe non-native accents and unnatural prosody. Sample A is overly breathy, while Sample B is robotic and monotone. Neither is a good representation of the target language." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects vowels in 'temps' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects multiple words.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is very flat and robotic, lacking natural conversational flow.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, with 'fenêtre' sounding like it has an added 's'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'fenêtre' and 'tranquille'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is overly whispered and breathy rather than naturally warm.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making some words difficult to parse.", "Both samples exhibit a very strong non-native (English) accent, making evaluation of native French prosody difficult." ] } }, { "id": "f09-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-de-DE-A.wav", "model": "s2-pro", "duration": 20.1288 }, "b": { "src": "audio/f09-de-DE-B.wav", "model": "s2.1-pro", "duration": 19.1303 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text accurately with appropriate pacing and tone. The differences in naturalness and audio quality are negligible, resulting in a tie.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples deliver the text accurately with appropriate pacing and tone. The differences in naturalness and audio quality are negligible, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples deliver the text with a very similar, breathy tone and a noticeable English accent that impacts the naturalness of the German pronunciation. Neither sample clearly outperforms the other." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural breath sounds and pacing.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on words like 'reserviert' and 'Keks'.", "severity": "major", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Minor stiffness in the transitions between sentences.", "severity": "minor", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent on words like 'reserviert' and 'Keks'.", "severity": "major", "timestamp_seconds": 6.0 } ] }, "limitations": [ "Both samples exhibit similar baseline audio quality and delivery styles, making differentiation difficult.", "Both samples feature a strong non-native accent that affects pronunciation." ] } }, { "id": "f09-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-pt-BR-A.wav", "model": "s2.1-pro", "duration": 18.7355 }, "b": { "src": "audio/f09-pt-BR-B.wav", "model": "s2-pro", "duration": 20.1055 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples feature a heavily whispered delivery with low volume and noticeable background hiss. Neither sample clearly outperforms the other.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a heavily whispered delivery with low volume and noticeable background hiss. Neither sample clearly outperforms the other." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples adopt a heavily whispered delivery that limits intelligibility and fails to capture the requested warm and teasing dynamics. They are functionally equivalent in quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and low volume throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is entirely whispered rather than warm and reassuring.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and low volume throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is entirely whispered rather than warm and reassuring.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Evaluation limited by extremely low volume and whispered delivery in both samples.", "Both samples are extremely quiet and whispered, making detailed phonetic analysis difficult." ] } }, { "id": "f09-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f09-ar-A.wav", "model": "s2-pro", "duration": 21.2665 }, "b": { "src": "audio/f09-ar-B.wav", "model": "s2.1-pro", "duration": 20.4538 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody and audio distortion. Sample B is slightly more intelligible despite the artifacts.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from severe robotic prosody and audio distortion. Sample B is slightly more intelligible despite the artifacts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from muffled audio quality and background hiss. The performances are very similar, with neither successfully capturing the requested emotional shift." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 2 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 2 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Heavy background noise and clicking throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Speech is highly disjointed and unnatural.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Muffled audio quality with noticeable background hiss throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background noise and distortion.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Speech is disjointed and lacks natural flow.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Muffled audio quality with noticeable background hiss throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit severe audio artifacts and robotic prosody, making them difficult to evaluate for subtle expressive cues.", "Arabic pronunciation nuances may be partially obscured by the low audio quality." ] } } ] }, { "id": "f10", "name": "Jami", "slug": "jami-b01a37", "gender": "Female", "rank": 10, "portrait": "portraits/f10.webp", "language": "8 languages", "popularity": { "callers": 78, "calls": 85 }, "pairs": [ { "id": "f10-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-zh-CN-A.wav", "model": "s2-pro", "duration": 16.0885 }, "b": { "src": "audio/f10-zh-CN-B.wav", "model": "s2.1-pro", "duration": 14.7417 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples have a heavy non-native accent that impacts naturalness and expressiveness. Sample A is slightly preferred for having marginally better flow and less choppy phrasing than Sample B.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples have a heavy non-native accent that impacts naturalness and expressiveness. Sample A is slightly preferred for having marginally better flow and less choppy phrasing than Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples have a noticeable non-native accent. Sample B is preferred because it avoids the pronunciation error present in Sample A." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent affects tone and rhythm throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing tones due to unnatural prosody.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and robotic, lacking the requested warmth.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent with incorrect tones and choppy phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Does not sound warm or gently teasing; delivery is stiff.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces '不着急' incorrectly, sounding like 'du zhao ji'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples feature a very strong non-native accent for Mandarin Chinese.", "Both samples feature a strong non-native accent, which affects naturalness and intelligibility." ] } }, { "id": "f10-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-ja-JP-A.wav", "model": "s2-pro", "duration": 17.3191 }, "b": { "src": "audio/f10-ja-JP-B.wav", "model": "s2.1-pro", "duration": 18.3176 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A is preferred due to better overall prosody and fewer pronunciation errors. Sample B suffers from multiple severe mispronunciations of basic words, making it less usable.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to better overall prosody and fewer pronunciation errors. Sample B suffers from multiple severe mispronunciations of basic words, making it less usable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from clear pronunciation errors on the first word and exhibit unnatural prosody. Neither is clearly better." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Expected 'Isoganakute' (急がなくて), heard 'Soganakute' with a missing initial vowel.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'soganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing and intonation are unnatural and choppy.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Expected 'Isoganakute' (急がなくて), heard 'Isaganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Expected 'Ashita no' or 'Asu no' (明日の), heard 'Isano'.", "severity": "major", "timestamp_seconds": 1.8 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '急がなくて' (isoganakute) as 'isaganakute'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日の' (ashita no / asu no) as 'isano'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "I am evaluating based on standard Japanese pronunciation; regional accents are considered but clear misreadings of kanji are penalized.", "Both samples exhibit strong foreign accents that affect naturalness and intelligibility." ] } }, { "id": "f10-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-ko-KR-A.wav", "model": "s2.1-pro", "duration": 16.8083 }, "b": { "src": "audio/f10-ko-KR-B.wav", "model": "s2-pro", "duration": 16.7386 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a more natural flow and better captures the intended warmth and gentle teasing, whereas Sample A sounds slightly more robotic.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural flow and better captures the intended warmth and gentle teasing, whereas Sample A sounds slightly more robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A has slightly more natural phrasing than Sample B, though both suffer from robotic prosody. Sample B's pitch contours are more disruptive." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing and pauses between sentences.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Very choppy phrasing and unnatural pitch contours.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural phrasing and intonation throughout the sample.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native Korean speaker, but robotic prosody is clearly audible." ] } }, { "id": "f10-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-es-ES-A.wav", "model": "s2.1-pro", "duration": 16.3207 }, "b": { "src": "audio/f10-es-ES-B.wav", "model": "s2-pro", "duration": 16.6457 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe English accent when speaking Spanish, making them sound unnatural for the requested locale. They are tied in quality.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent when speaking Spanish, making them sound unnatural for the requested locale. They are tied in quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples apply English phonetics to the Spanish text, resulting in a severe foreign accent and mispronunciations, such as sounding the silent 'h' in 'He'. Neither is acceptable for a native Spanish locale." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' and uses English vowel sounds throughout.", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' and uses English vowel sounds throughout.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples feature a severe non-native (American English) accent that heavily impacts naturalness.", "Both samples exhibit a severe English accent, making it difficult to assess native Spanish prosody." ] } }, { "id": "f10-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-fr-FR-A.wav", "model": "s2.1-pro", "duration": 14.3005 }, "b": { "src": "audio/f10-fr-FR-B.wav", "model": "s2-pro", "duration": 15.3687 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe English accent that heavily impacts pronunciation and naturalness. Neither is clearly better.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent that heavily impacts pronunciation and naturalness. Neither is clearly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers slightly better prosody and flow. Both samples suffer from a heavy non-native accent, but A is more disjointed." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on 'Prends ton temps' and 'biscuit'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "major", "timestamp_seconds": 0.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent on 'Prends ton temps' and 'biscuit'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'Prends ton temps' sounds like 'Prince ton ton'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed phrasing and unnatural pauses.", "severity": "major", "timestamp_seconds": 5.0 } ] }, "limitations": [ "Both samples exhibit a very strong non-native accent, making it difficult to assess subtle prosodic features.", "Both samples feature a strong non-native (English) accent, affecting naturalness." ] } }, { "id": "f10-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-de-DE-A.wav", "model": "s2-pro", "duration": 17.0637 }, "b": { "src": "audio/f10-de-DE-B.wav", "model": "s2.1-pro", "duration": 16.9244 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples mispronounce the first sentence as English words. Sample A is preferred because it pronounces the time correctly, whereas Sample B inserts an unnatural pause in the number.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples mispronounce the first sentence as English words. Sample A is preferred because it pronounces the time correctly, whereas Sample B inserts an unnatural pause in the number." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from an identical, severe English accent that distorts the German text, making words like 'Lass dir Zeit' sound like 'Last dear sight'. Neither is clearly better." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "wrong_language", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the German text.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced with a strong English accent, sounding like 'Last dear sight'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ], "B": [ { "order": "AB", "category": "wrong_language", "evidence": "Pronounces 'Lass dir Zeit' as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause in the number '19 Uhr 5 und 40'.", "severity": "minor", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced with a strong English accent, sounding like 'Last dear sight'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "pronunciation", "evidence": "'Keks' is pronounced like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 } ] }, "limitations": [ "Both models have a strong non-native accent.", "Both samples feature a very strong English accent that heavily distorts German pronunciation." ] } }, { "id": "f10-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-pt-BR-A.wav", "model": "s2-pro", "duration": 17.4585 }, "b": { "src": "audio/f10-pt-BR-B.wav", "model": "s2.1-pro", "duration": 16.2278 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a strong non-native accent that affects pronunciation and naturalness. They are equally intelligible and follow the text accurately, resulting in a tie.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a strong non-native accent that affects pronunciation and naturalness. They are equally intelligible and follow the text accurately, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples have good text fidelity and intelligibility but suffer from a noticeable non-native accent and somewhat stiff prosody. Neither clearly outperforms the other in expressiveness or naturalness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the sample, affecting words like 'amanhã' and 'janela'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stiff and slightly unnatural cadence.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout the sample, affecting words like 'amanhã' and 'janela'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stiff and slightly unnatural cadence.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent, making it difficult to assess native-level prosody.", "Both samples exhibit a non-native accent for Brazilian Portuguese." ] } }, { "id": "f10-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f10-ar-A.wav", "model": "s2-pro", "duration": 21.7077 }, "b": { "src": "audio/f10-ar-B.wav", "model": "s2.1-pro", "duration": 17.8764 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe foreign accent that heavily distorts Arabic phonemes, making the text nearly unintelligible. Neither is usable for the target language.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe foreign accent that heavily distorts Arabic phonemes, making the text nearly unintelligible. Neither is usable for the target language." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe foreign accent and robotic prosody. Neither successfully captures the requested warm or teasing delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent distorts most words, e.g., 'للاستعجال' and 'مقعدين'.", "severity": "blocking", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe foreign accent affects word clarity.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural intonation and disjointed rhythm.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe foreign accent distorts most words, e.g., 'للاستعجال' and 'مقعدين'.", "severity": "blocking", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is disjointed and lacks natural Arabic cadence.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Non-native speaker evaluation, but the severe foreign accent and phoneme distortion are clearly audible.", "Both samples feature a very strong foreign accent, making some words difficult to parse." ] } } ] }, { "id": "f11", "name": "Vivienne Brooks", "slug": "vivienne-brooks-ed4cff", "gender": "Female", "rank": 11, "portrait": "portraits/f11.webp", "language": "8 languages", "popularity": { "callers": 39, "calls": 50 }, "pairs": [ { "id": "f11-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-zh-CN-A.wav", "model": "s2.1-pro", "duration": 14.8114 }, "b": { "src": "audio/f11-zh-CN-B.wav", "model": "s2-pro", "duration": 15.9492 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural conversational tone that fits the warm and gently teasing directions well. Sample B is overly breathy and slow, making it sound less natural.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural conversational tone that fits the warm and gently teasing directions well. Sample B is overly breathy and slow, making it sound less natural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a slightly more natural and expressive delivery, better capturing the intended warmth and teasing tone. Sample B is clear but slightly flatter in its emotional transitions." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and slow delivery throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f11-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-ja-JP-A.wav", "model": "s2-pro", "duration": 17.7371 }, "b": { "src": "audio/f11-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.8037 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A delivers the text accurately, though the expressiveness is somewhat flat. Sample B has better emotional delivery but makes a confusing pronunciation error, reading '明日' (tomorrow) as 'asa' (morning).", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A delivers the text accurately, though the expressiveness is somewhat flat. Sample B has better emotional delivery but makes a confusing pronunciation error, reading '明日' (tomorrow) as 'asa' (morning)." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B reads the text accurately. Sample A has slightly better expressiveness but misreads a word, making B the preferred choice." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The tone is somewhat serious and lacks the requested warmth and gentle teasing.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日' (ashita) as 'asa', resulting in the confusing phrase 'asa no yoru' (morning's night).", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Misreads '明日' (ashita) as '朝' (asa).", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Evaluator is not a native speaker of Japanese, but can identify clear pronunciation errors." ] } }, { "id": "f11-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-ko-KR-A.wav", "model": "s2.1-pro", "duration": 18.1086 }, "b": { "src": "audio/f11-ko-KR-B.wav", "model": "s2-pro", "duration": 17.83 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural and expressive delivery that better matches the warm and gently teasing directions. Sample B sounds somewhat flat and serious in comparison.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery that better matches the warm and gently teasing directions. Sample B sounds somewhat flat and serious in comparison." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly more natural pacing and better captures the requested warm and teasing tones. Sample B is also good but feels a bit more uniform in delivery." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is somewhat flat and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle accent nuances may be missed." ] } }, { "id": "f11-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.9275 }, "b": { "src": "audio/f11-es-ES-B.wav", "model": "s2-pro", "duration": 16.994 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has better pacing and more natural phrasing. Sample B is overly slow and disjointed, though both suffer from a noticeable non-native accent.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A has better pacing and more natural phrasing. Sample B is overly slow and disjointed, though both suffer from a noticeable non-native accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers more natural pacing and phrasing. Sample A is overly slow and robotic, failing to capture the requested emotional shifts." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The 'H' in 'He reservado' is audibly aspirated, which is incorrect for Spanish.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The emotional shifts are subtle and could be more pronounced.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is very slow and disjointed throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Aspirated 'H' in 'He reservado'.", "severity": "minor", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Speech is very slow and disjointed, lacking natural flow.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the warm or teasing tones requested.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native (English) accent, making it difficult to assess native Spanish (Spain) nuances." ] } }, { "id": "f11-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-fr-FR-A.wav", "model": "s2.1-pro", "duration": 14.672 }, "b": { "src": "audio/f11-fr-FR-B.wav", "model": "s2-pro", "duration": 15.717 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly better flow and more natural phrasing. Sample B sounds more disjointed and robotic, making it less engaging.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly better flow and more natural phrasing. Sample B sounds more disjointed and robotic, making it less engaging." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides a much more natural and fluid delivery. Sample A suffers from disjointed pacing and robotic prosody that detracts from the intended warmth." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent affects the naturalness of some vowels and consonants.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Slight non-native accent, though it does not impede intelligibility.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Speech is choppy and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent impacts the pronunciation of words like 'tranquille'.", "severity": "minor", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed pacing and unnatural pauses throughout the utterance.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit noticeable non-native accents, but evaluations focus on intelligibility and prosody rather than penalizing the accent itself.", "Evaluation of subtle regional accents is limited by the lack of a reference speaker." ] } }, { "id": "f11-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.7603 }, "b": { "src": "audio/f11-de-DE-B.wav", "model": "s2-pro", "duration": 17.1798 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers more natural prosody and better captures the requested warm and teasing tones. Sample B sounds slightly more robotic and flat in its delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers more natural prosody and better captures the requested warm and teasing tones. Sample B sounds slightly more robotic and flat in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly smoother prosody and more natural pacing. Sample A sounds a bit more disjointed in its phrasing, though both handle the text accurately." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The shift to gentle teasing is not very pronounced.", "severity": "minor", "timestamp_seconds": 13.0 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat stiff and lack natural flow.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly disjointed phrasing and unnatural pauses.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f11-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-pt-BR-A.wav", "model": "s2.1-pro", "duration": 19.525 }, "b": { "src": "audio/f11-pt-BR-B.wav", "model": "s2-pro", "duration": 18.2479 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples fail to use the requested Brazilian Portuguese locale, instead sounding like European Portuguese. Sample B is slightly preferred for a marginally more natural delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, instead sounding like European Portuguese. Sample B is slightly preferred for a marginally more natural delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese. Sample A is preferred for its more natural prosody and better expressiveness, whereas Sample B sounds robotic." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "wrong_language", "evidence": "The pronunciation and accent are distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "The pronunciation and accent are distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is stiff and lacks natural cadence.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "wrong_language", "evidence": "The pronunciation and accent are distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "The pronunciation and accent are distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale.", "Both samples produce European Portuguese instead of the requested Brazilian Portuguese locale." ] } }, { "id": "f11-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f11-ar-A.wav", "model": "s2.1-pro", "duration": 19.5018 }, "b": { "src": "audio/f11-ar-B.wav", "model": "s2-pro", "duration": 17.5281 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred because it reads the text much more accurately. Sample B skips a phrase entirely and misreads the time.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A is preferred because it reads the text much more accurately. Sample B skips a phrase entirely and misreads the time." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B is preferred because it preserves the target text much better. Sample A omits a phrase entirely and misreads the time." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "2.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafitha'.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ], "B": [ { "order": "AB", "category": "text_omission", "evidence": "Skips 'يمكننا الجلوس قرب' entirely.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "AB", "category": "text_substitution", "evidence": "Misreads the time '7:45 مساءً'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "text_omission", "evidence": "Omitted 'يمكننا الجلوس قرب' before 'النافذة'.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "text_substitution", "evidence": "Misread the time '7:45 مساءً' as 'سبعة وأربعين...'.", "severity": "major", "timestamp_seconds": 4.0 } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, but can identify clear text omissions and pronunciation errors." ] } } ] }, { "id": "f12", "name": "Nora", "slug": "nora-d25df2", "gender": "Female", "rank": 12, "portrait": "portraits/f12.webp", "language": "8 languages", "popularity": { "callers": 39, "calls": 47 }, "pairs": [ { "id": "f12-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-zh-CN-A.wav", "model": "s2.1-pro", "duration": 17.5745 }, "b": { "src": "audio/f12-zh-CN-B.wav", "model": "s2-pro", "duration": 17.6442 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A delivers a more natural and expressive performance, better capturing the requested warm and teasing tones. Sample B sounds slightly stiff and robotic in its prosody.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A delivers a more natural and expressive performance, better capturing the requested warm and teasing tones. Sample B sounds slightly stiff and robotic in its prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples are highly intelligible but lack the requested emotional shifts, sounding somewhat robotic. Sample B has slightly better phrasing and flow." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and does not fully capture the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff cadence and unnatural pausing throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested warm and teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f12-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-ja-JP-A.wav", "model": "s2.1-pro", "duration": 18.8749 }, "b": { "src": "audio/f12-ja-JP-B.wav", "model": "s2-pro", "duration": 19.7108 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A provides a much more natural and expressive delivery, capturing the intended warm and teasing tones effectively. Sample B suffers from somewhat robotic prosody and awkward phrasing.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A provides a much more natural and expressive delivery, capturing the intended warm and teasing tones effectively. Sample B suffers from somewhat robotic prosody and awkward phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural conversational flow and better captures the requested warm and teasing tones. Sample A sounds slightly robotic and disjointed in its phrasing." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The phrasing and pacing sound somewhat disjointed and unnatural.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The phrasing and cadence feel slightly unnatural and disjointed.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional accents in Japanese is limited." ] } }, { "id": "f12-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-ko-KR-A.wav", "model": "s2-pro", "duration": 20.8253 }, "b": { "src": "audio/f12-ko-KR-B.wav", "model": "s2.1-pro", "duration": 19.8733 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B has a more natural conversational pace and better captures the subtle shift in tone. Sample A is slightly too breathy and slow, making it feel less natural.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B has a more natural conversational pace and better captures the subtle shift in tone. Sample A is slightly too breathy and slow, making it feel less natural." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A delivers a more natural and appropriate tone for the requested directions. Sample B is overly breathy and whispered, which feels unnatural for the context." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is slightly slow and overly breathy throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and whispered throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [ "I am not a native Korean speaker, so subtle regional accent variations might not be fully captured.", "I am not a native Korean speaker, so subtle nuances in regional accents or advanced prosody may not be fully evaluated." ] } }, { "id": "f12-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-es-ES-A.wav", "model": "s2-pro", "duration": 17.9461 }, "b": { "src": "audio/f12-es-ES-B.wav", "model": "s2.1-pro", "duration": 17.8532 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe English accent, failing to produce natural Spanish (Spain) pronunciation. They are tied as both are equally unnatural for the target locale.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent, failing to produce natural Spanish (Spain) pronunciation. They are tied as both are equally unnatural for the target locale." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a very strong English accent that detracts from the Spanish (Spain) locale. Sample A is slightly preferred due to a marginally more natural cadence." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the entire sample, e.g., 'galleta' and 'reservado'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, affecting vowels and consonants (e.g., 'galleta').", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff and mechanical pacing compared to A.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout, with awkward pronunciation of 'galleta'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and does not match the requested emotional cues.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong English accent throughout, affecting vowels and consonants (e.g., 'galleta').", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong English accent, making it difficult to assess native-like prosody.", "Evaluated by a non-native speaker, though the strong English accent in both samples is clearly identifiable." ] } }, { "id": "f12-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-fr-FR-A.wav", "model": "s2-pro", "duration": 17.5745 }, "b": { "src": "audio/f12-fr-FR-B.wav", "model": "s2.1-pro", "duration": 16.3439 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers smoother phrasing and better overall flow. Sample A suffers from disjointed pauses and a more robotic cadence.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers smoother phrasing and better overall flow. Sample A suffers from disjointed pauses and a more robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a heavy English accent with significant pronunciation errors on words like 'tranquille' and 'biscuit'. Neither sounds like a native French speaker, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Disjointed phrasing with unnatural pauses between clauses.", "severity": "major", "timestamp_seconds": 6.0 }, { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects vowel sounds throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent, mispronouncing 'tranquille' as 'tran-kwil' and 'biscuit' as 'bis-kwee'.", "severity": "major", "timestamp_seconds": 11.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent affects vowel sounds throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, mispronouncing 'biscuit' as 'bis-kwee'.", "severity": "major", "timestamp_seconds": 14.0 } ] }, "limitations": [ "Both samples feature a strong non-native accent, which affects naturalness but does not completely impede intelligibility.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f12-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-de-DE-A.wav", "model": "s2.1-pro", "duration": 18.4801 }, "b": { "src": "audio/f12-de-DE-B.wav", "model": "s2-pro", "duration": 18.9213 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples deliver the text accurately with clear pronunciation and good audio quality. The expressiveness is somewhat neutral in both, missing the nuanced shifts requested, making them equally usable.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with clear pronunciation and good audio quality. The expressiveness is somewhat neutral in both, missing the nuanced shifts requested, making them equally usable." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly intelligible and accurate to the text, but share a similar slightly robotic prosody. Neither stands out as significantly better in naturalness or expressiveness." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery remains mostly neutral, missing the requested shift to gentle teasing.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The phrasing and rhythm are somewhat mechanical.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is generally neutral and does not clearly convey the requested warm or teasing tones.", "severity": "minor", "timestamp_seconds": 14.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall cadence is slightly stiff and lacks natural human flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a slightly robotic cadence." ] } }, { "id": "f12-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-pt-BR-A.wav", "model": "s2.1-pro", "duration": 18.2015 }, "b": { "src": "audio/f12-pt-BR-B.wav", "model": "s2-pro", "duration": 20.9647 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred because it maintains accurate text fidelity. Sample B incorrectly pluralizes a word, and both fail to use the requested Brazilian Portuguese accent.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred because it maintains accurate text fidelity. Sample B incorrectly pluralizes a word, and both fail to use the requested Brazilian Portuguese accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese. Sample B is preferred because it avoids the text fidelity error found in Sample A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The accent sounds like European Portuguese, not Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_addition", "evidence": "Speaker says 'noites' instead of 'noite'.", "severity": "major", "timestamp_seconds": 7.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Heard 'noites' instead of 'noite'.", "severity": "minor", "timestamp_seconds": 6.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The accent sounds like European Portuguese, not Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a European Portuguese accent rather than the requested Brazilian Portuguese locale.", "I am evaluating the accent based on general phonetic differences between European and Brazilian Portuguese." ] } }, { "id": "f12-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f12-ar-A.wav", "model": "s2.1-pro", "duration": 21.4523 }, "b": { "src": "audio/f12-ar-B.wav", "model": "s2-pro", "duration": 19.9894 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit similar levels of robotic prosody and unnatural pausing. Text fidelity is maintained in both, but neither sounds fully natural or highly expressive.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar levels of robotic prosody and unnatural pausing. Text fidelity is maintained in both, but neither sounds fully natural or highly expressive." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from similar robotic prosody and unnatural phrasing, particularly an awkward pause before 'النافذة'. Neither successfully captures the requested emotional shifts." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses and somewhat mechanical cadence throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and does not reflect the requested emotional cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pauses, such as before 'النافذة', and mechanical delivery.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm or teasing tones, remaining flat.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, but can assess general intelligibility and text fidelity.", "Both samples exhibit similar unnatural pauses and robotic prosody, making differentiation difficult." ] } } ] }, { "id": "f13", "name": "Clara", "slug": "clara-a46da7", "gender": "Female", "rank": 13, "portrait": "portraits/f13.webp", "language": "8 languages", "popularity": { "callers": 38, "calls": 53 }, "pairs": [ { "id": "f13-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-zh-CN-A.wav", "model": "s2.1-pro", "duration": 15.2061 }, "b": { "src": "audio/f13-zh-CN-B.wav", "model": "s2-pro", "duration": 14.9275 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with good intelligibility. They both exhibit slight robotic prosody and limited expressiveness, making them comparable in overall quality.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with good intelligibility. They both exhibit slight robotic prosody and limited expressiveness, making them comparable in overall quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural cadence and better captures the requested warm and reassuring tone. Sample B is intelligible but sounds slightly more robotic and flat." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly choppy pacing and unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat and mechanical cadence throughout the sample.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and phrasing.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f13-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-ja-JP-A.wav", "model": "s2-pro", "duration": 18.5963 }, "b": { "src": "audio/f13-ja-JP-B.wav", "model": "s2.1-pro", "duration": 20.8718 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe robotic prosody and unnatural intonation. Sample A is preferred because it avoids the extreme and disruptive pitch fluctuations present in Sample B.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and unnatural intonation. Sample A is preferred because it avoids the extreme and disruptive pitch fluctuations present in Sample B." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe robotic prosody and strong non-native accents. Neither successfully delivers a natural Japanese performance, resulting in a tie." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The entire utterance has a very unnatural, robotic cadence with incorrect pitch accents.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stilted and unnatural cadence throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent impacts the delivery.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Extreme and unnatural pitch fluctuations throughout the recording, sounding almost sing-song in a disruptive way.", "severity": "blocking", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly unnatural pitch shifts and cadence throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent affects overall intelligibility and naturalness.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit severe non-native or robotic prosody, making naturalness evaluation challenging.", "Both samples exhibit strong foreign accents and unnatural prosody, making them difficult to evaluate for native-level expressiveness." ] } }, { "id": "f13-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-ko-KR-A.wav", "model": "s2.1-pro", "duration": 18.6195 }, "b": { "src": "audio/f13-ko-KR-B.wav", "model": "s2-pro", "duration": 17.3424 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples are highly intelligible and accurate to the text. Sample B is preferred for slightly better pacing and naturalness, though neither fully captures the requested emotional shifts.", "confidence": 0.65, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Both samples are highly intelligible and accurate to the text. Sample B is preferred for slightly better pacing and naturalness, though neither fully captures the requested emotional shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A is slightly preferred due to marginally more natural phrasing and pacing. Both samples suffer from somewhat robotic prosody and limited expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff delivery with unnatural pauses between phrases.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and gently teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is slightly disjointed with unnatural pauses.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery remains mostly neutral, missing the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing and intonation feel somewhat uniform and synthetic throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Non-native speaker; evaluating based on general prosody, clarity, and expected emotional delivery.", "Both samples exhibit slightly robotic prosody, making it difficult to strongly prefer one over the other." ] } }, { "id": "f13-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-es-ES-A.wav", "model": "s2-pro", "duration": 15.09 }, "b": { "src": "audio/f13-es-ES-B.wav", "model": "s2.1-pro", "duration": 15.6009 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a heavy English accent when speaking Spanish. Sample A is preferred for having slightly more natural intonation, whereas Sample B sounds more robotic and flat.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples feature a heavy English accent when speaking Spanish. Sample A is preferred for having slightly more natural intonation, whereas Sample B sounds more robotic and flat." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to produce acceptable Spanish pronunciation, sounding like an English speaker reading Spanish phonetically. Neither is usable for the target locale." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'reservado' and 'sitios'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'sitios' pronounced as 'shiti-os', 'galleta' as 'gallieta'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and lacks the requested emotional shifts.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout the recording.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks natural cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm or teasing directions.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent throughout, e.g., 'tengas' pronounced as 'tangas', 'galleta' as 'gallieta'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, lacking the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody.", "Both samples exhibit a severe non-native (English) accent, making the Spanish pronunciation highly unnatural and difficult to understand." ] } }, { "id": "f13-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-fr-FR-A.wav", "model": "s2-pro", "duration": 16.8083 }, "b": { "src": "audio/f13-fr-FR-B.wav", "model": "s2.1-pro", "duration": 15.508 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples exhibit a strong non-native accent. Sample B is preferred for its slightly more fluid phrasing, whereas Sample A sounds more disjointed.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a strong non-native accent. Sample B is preferred for its slightly more fluid phrasing, whereas Sample A sounds more disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a heavy non-native accent and robotic prosody. Sample A is preferred for having slightly better pacing and flow, whereas Sample B sounds more disjointed." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, e.g., 'fenêtre' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Disjointed pacing with unnatural pauses between phrases.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is disjointed with unnatural pauses between phrases.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent affects pronunciation throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat disjointed and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent affects words like 'tranquille' and 'biscuit'.", "severity": "minor", "timestamp_seconds": 10.0 } ] }, "limitations": [ "Both samples feature a strong non-native accent, making it difficult to assess native-level prosody.", "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f13-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-de-DE-A.wav", "model": "s2-pro", "duration": 16.4135 }, "b": { "src": "audio/f13-de-DE-B.wav", "model": "s2.1-pro", "duration": 17.0173 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe English accent, mispronouncing words like 'Keks' as 'cakes', and lack the requested emotional delivery. Neither is suitable for native German speech.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe English accent, mispronouncing words like 'Keks' as 'cakes', and lack the requested emotional delivery. Neither is suitable for native German speech." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples mispronounce 'Keks' as the English word 'cakes'. Sample B is preferred because it has slightly more natural prosody and phrasing compared to Sample A's robotic cadence." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, 'Keks' pronounced as 'cakes'.", "severity": "major", "timestamp_seconds": 14.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "minor", "timestamp_seconds": 14.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, 'Keks' pronounced as 'cakes'.", "severity": "major", "timestamp_seconds": 15.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm, reassuring, and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'Keks' as the English word 'cakes'.", "severity": "minor", "timestamp_seconds": 15.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing and cadence sound somewhat unnatural and robotic.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples mispronounce 'Keks' as the English word 'cakes'." ] } }, { "id": "f13-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-pt-BR-A.wav", "model": "s2-pro", "duration": 16.251 }, "b": { "src": "audio/f13-pt-BR-B.wav", "model": "s2.1-pro", "duration": 17.3424 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples fail to use the requested Brazilian Portuguese accent. However, Sample A sounds much more natural and fluent, whereas Sample B has a strong non-native accent and robotic prosody.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese accent. However, Sample A sounds much more natural and fluent, whereas Sample B has a strong non-native accent and robotic prosody." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred for its slightly better naturalness and prosody, although it exhibits a strong European Portuguese accent rather than the requested Brazilian locale. Sample A sounds highly robotic and non-native." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The accent is distinctly European Portuguese (pt-PT) rather than the requested Brazilian Portuguese (pt-BR).", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and gently teasing emotional shifts.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker uses a non-native or European Portuguese accent instead of Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is stiff and lacks natural phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat, mechanical, and lacks the requested warmth and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The accent sounds non-native, with unnatural vowel reductions and consonant pronunciations for Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a European Portuguese or non-native accent rather than the requested Brazilian Portuguese locale.", "Both samples exhibit non-target accents; Sample B sounds distinctly like European Portuguese rather than Brazilian Portuguese." ] } }, { "id": "f13-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f13-ar-A.wav", "model": "s2-pro", "duration": 19.7805 }, "b": { "src": "audio/f13-ar-B.wav", "model": "s2.1-pro", "duration": 20.0127 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit robotic prosody and lack the requested emotional shifts. Sample A is slightly preferred for having fewer awkward pauses and a marginally smoother flow.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples exhibit robotic prosody and lack the requested emotional shifts. Sample A is slightly preferred for having fewer awkward pauses and a marginally smoother flow." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are intelligible and capture the intended tone, but both suffer from unnatural phrasing and pauses, particularly around 'قرب النافذة'. Neither is clearly superior." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks the requested warm and teasing emotional shifts.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_omission", "evidence": "Missing definite article in 'للاستعجال', pronounced as 'لاستعجال'.", "severity": "minor", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech is flat and contains awkward pauses, such as between 'qurb' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to convey the warm, reassuring, and gently teasing tones requested.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'قرب' and 'النافذة'.", "severity": "minor", "timestamp_seconds": 7.5 } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, so subtle pronunciation or accent nuances may not be fully evaluated.", "Both samples exhibit unnatural pauses that affect the flow of the Arabic text." ] } } ] }, { "id": "f14", "name": "Yumi", "slug": "yumi-94c764", "gender": "Female", "rank": 14, "portrait": "portraits/f14.webp", "language": "8 languages", "popularity": { "callers": 34, "calls": 45 }, "pairs": [ { "id": "f14-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-zh-CN-A.wav", "model": "s2.1-pro", "duration": 13.9755 }, "b": { "src": "audio/f14-zh-CN-B.wav", "model": "s2-pro", "duration": 14.7417 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers a slightly more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B sounds more robotic and stiff in its phrasing.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a slightly more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B sounds more robotic and stiff in its phrasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples feature a strong non-native accent with pervasive tone errors. Sample B is slightly preferred for its marginally better pacing and flow." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent with incorrect tones throughout the utterance.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Stiff phrasing and cadence throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent with incorrect tones throughout the utterance.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator's native language is not Mandarin, but can assess general prosody and text fidelity." ] } }, { "id": "f14-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-ja-JP-A.wav", "model": "s2-pro", "duration": 17.4352 }, "b": { "src": "audio/f14-ja-JP-B.wav", "model": "s2.1-pro", "duration": 16.3207 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred due to clearer articulation and better intelligibility. Sample B slurs some words early in the recording.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred due to clearer articulation and better intelligibility. Sample B slurs some words early in the recording." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B accurately pronounces the Japanese text and follows the emotional cues. Sample A has severe pronunciation errors, particularly with the date and time." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "2.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout, affecting prosody.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Slurred pronunciation around 'Ashita no yoru'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Non-native accent affects overall prosody and naturalness.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日の夜7時45分' (Ashita no yoru shichiji yonjugofun) as 'Asa no yashiti anjugofun'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, which affects naturalness.", "I am not a native Japanese speaker, but the pronunciation errors in Sample A are clear." ] } }, { "id": "f14-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-ko-KR-A.wav", "model": "s2.1-pro", "duration": 16.1349 }, "b": { "src": "audio/f14-ko-KR-B.wav", "model": "s2-pro", "duration": 16.9476 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has slightly more natural phrasing and pacing. Sample B exhibits a slightly more robotic cadence.", "confidence": 0.7, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A has slightly more natural phrasing and pacing. Sample B exhibits a slightly more robotic cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample B offers slightly smoother phrasing and avoids the distracting click present in Sample A. Both samples have a noticeable non-native accent." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural intonation and pacing due to heavy accent.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff and unnatural cadence throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "click_or_glitch", "evidence": "Audible click or sharp breath sound.", "severity": "minor", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stilted pacing and unnatural intonation throughout.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native Korean speaker, so subtle accent nuances may be missed.", "Both samples exhibit a strong non-native accent, making subtle prosodic evaluation difficult." ] } }, { "id": "f14-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-es-ES-A.wav", "model": "s2.1-pro", "duration": 14.8578 }, "b": { "src": "audio/f14-es-ES-B.wav", "model": "s2-pro", "duration": 15.5776 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe English accent that heavily impacts naturalness and pronunciation in Spanish. Neither is clearly better.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent that heavily impacts naturalness and pronunciation in Spanish. Neither is clearly better." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a severe English accent, mispronouncing Spanish phonemes (e.g., pronouncing the silent 'h' in 'he'). Neither sounds like a native Spanish speaker." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'He reservado' pronounced with an English 'h' and 'r'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' and uses English phonemes throughout.", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout, e.g., 'He reservado' pronounced with an English 'h' and 'r'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces the silent 'h' in 'He reservado' and uses English phonemes throughout.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples feature a very strong English accent, making it difficult to assess native Spanish prosody.", "Both samples exhibit a very strong English accent, making it difficult to assess native Spanish prosody." ] } }, { "id": "f14-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-fr-FR-A.wav", "model": "s2-pro", "duration": 14.765 }, "b": { "src": "audio/f14-fr-FR-B.wav", "model": "s2.1-pro", "duration": 15.717 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a severe English accent that heavily distorts French pronunciation, making them equally unsuitable for a native French locale.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples feature a severe English accent that heavily distorts French pronunciation, making them equally unsuitable for a native French locale." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe English accent that heavily distorts the French pronunciation. Sample A is slightly preferred as it has marginally better prosody and expressiveness compared to the flatter delivery of B." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent distorts vowels, e.g., 'réservé' sounds like 'réservi'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "pronunciation", "evidence": "'19 h 45' is pronounced with 'air' instead of 'heures'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Severe English accent, e.g., 'réservé' sounds like 'rezervi'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Delivery is somewhat flat and mechanical.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout the recording.", "severity": "major", "timestamp_seconds": 1.0 }, { "order": "AB", "category": "pronunciation", "evidence": "'19 h 45' is pronounced with 'air' instead of 'heures'.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'tranquille' pronounced with a hard 'L'.", "severity": "major", "timestamp_seconds": 11.0 } ] }, "limitations": [ "Both samples feature a very strong non-native (English) accent that severely impacts French pronunciation." ] } }, { "id": "f14-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-de-DE-A.wav", "model": "s2.1-pro", "duration": 14.9042 }, "b": { "src": "audio/f14-de-DE-B.wav", "model": "s2-pro", "duration": 15.7402 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from the exact same severe mispronunciation at the beginning, reading the German phrase as English words. The remaining German is intelligible but accented in both.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from the exact same severe mispronunciation at the beginning, reading the German phrase as English words. The remaining German is intelligible but accented in both." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples severely mispronounce the opening phrase 'Lass dir Zeit' as English words and exhibit a very strong non-native accent throughout. They are equally flawed." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "2.0/5 mean text-fidelity score", "B": "2.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced as English words ('last year sight').", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced as English 'Last year sight'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "'Lass dir Zeit' is pronounced as English words ('last year sight').", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both models mispronounce the first sentence as English words." ] } }, { "id": "f14-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-pt-BR-A.wav", "model": "s2-pro", "duration": 15.9027 }, "b": { "src": "audio/f14-pt-BR-B.wav", "model": "s2.1-pro", "duration": 15.9956 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples fail to accurately reflect the requested Brazilian Portuguese locale, heavily utilizing European Portuguese pronunciation and cadence. They are tied due to this shared locale mismatch.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to accurately reflect the requested Brazilian Portuguese locale, heavily utilizing European Portuguese pronunciation and cadence. They are tied due to this shared locale mismatch." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples have a noticeable foreign accent, but Sample B provides slightly better pronunciation and intelligibility. Sample A struggles more with phonemes in words like 'juntos'." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Uses European Portuguese phonology (e.g., 'juntos' with a palatal fricative).", "severity": "major", "timestamp_seconds": 10.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable foreign accent throughout, though slightly clearer than A.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong European Portuguese accent (e.g., clipped vowels in 'noite', 'juntos', 'promete').", "severity": "major", "timestamp_seconds": 5.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent affects words like 'juntos' and 'janela'.", "severity": "major", "timestamp_seconds": 7.0 } ] }, "limitations": [ "Both samples exhibit strong European Portuguese pronunciation features despite the pt-BR locale request.", "Both samples exhibit a strong foreign accent, making it difficult to assess native-level naturalness." ] } }, { "id": "f14-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f14-ar-A.wav", "model": "s2-pro", "duration": 19.1767 }, "b": { "src": "audio/f14-ar-B.wav", "model": "s2.1-pro", "duration": 16.4832 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples suffer from a severe non-native accent that heavily distorts Arabic phonemes. Neither is usable for native Arabic speech.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe non-native accent that heavily distorts Arabic phonemes. Neither is usable for native Arabic speech." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples provide intelligible speech with acceptable audio quality. Neither sample clearly outperforms the other in expressiveness or naturalness." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "3.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe non-native accent distorts most words, e.g., 'للاستعجال' and 'مقعدين'.", "severity": "major", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe non-native accent distorts most words, e.g., 'للاستعجال' and 'قرب'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a severe non-native accent, making phoneme evaluation difficult.", "Non-native Arabic speaker; limited ability to detect subtle prosodic or dialectal nuances." ] } } ] }, { "id": "f15", "name": "Elena", "slug": "elena-50d35c", "gender": "Female", "rank": 15, "portrait": "portraits/f15.webp", "language": "8 languages", "popularity": { "callers": 34, "calls": 44 }, "pairs": [ { "id": "f15-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-zh-CN-A.wav", "model": "s2-pro", "duration": 18.4105 }, "b": { "src": "audio/f15-zh-CN-B.wav", "model": "s2.1-pro", "duration": 17.3424 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a much more natural and conversational delivery. Sample A is overly breathy and slow, which feels exaggerated and less suitable for a standard conversation.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides a much more natural and conversational delivery. Sample A is overly breathy and slow, which feels exaggerated and less suitable for a standard conversation." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a clearer and more natural delivery, though slightly stiff. Sample B is overly breathy and whispered, which detracts from intelligibility and does not fit the requested delivery well." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is overly breathy and slow, making it sound somewhat unnatural for a normal conversation.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The entire sample is spoken in a heavy, breathy whisper, which does not fit the warm or teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing and transitions between phrases.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle emotional shifts in Mandarin may vary by regional preference." ] } }, { "id": "f15-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.4786 }, "b": { "src": "audio/f15-ja-JP-B.wav", "model": "s2-pro", "duration": 21.3362 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is preferred because Sample A mispronounces a word, changing the meaning. Both samples suffer from unnatural, robotic prosody, but B is more accurate to the text.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred because Sample A mispronounces a word, changing the meaning. Both samples suffer from unnatural, robotic prosody, but B is more accurate to the text." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A delivers the text accurately with appropriate warmth. Sample B mispronounces '明日' (tomorrow) as 'asa' (morning), creating the confusing phrase 'morning's night'." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Misreads '明日' (ashita/asu) as 'asa' (morning), resulting in 'asa no yoru' which is contradictory.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pitch accents and phrasing throughout the utterance.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces '明日' (ashita/asu) as 'asa' (morning), resulting in 'asa no yoru'.", "severity": "major", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Speech is overly slow, breathy, and lacks natural Japanese pitch accent.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit noticeable foreign accents and unnatural pitch accents for Japanese." ] } }, { "id": "f15-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-ko-KR-A.wav", "model": "s2.1-pro", "duration": 18.7588 }, "b": { "src": "audio/f15-ko-KR-B.wav", "model": "s2-pro", "duration": 19.1071 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers a slightly more natural and expressive delivery, capturing the requested warm and teasing tones better than Sample B, which sounds a bit more monotonous.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers a slightly more natural and expressive delivery, capturing the requested warm and teasing tones better than Sample B, which sounds a bit more monotonous." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A delivers a more natural and expressive performance that better captures the requested warm and teasing tones. Sample B is intelligible but sounds slightly stiffer in its phrasing." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff phrasing throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [] } }, { "id": "f15-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-es-ES-A.wav", "model": "s2-pro", "duration": 19.0606 }, "b": { "src": "audio/f15-es-ES-B.wav", "model": "s2.1-pro", "duration": 18.4801 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has slightly better pacing and expressiveness, though both exhibit a noticeable non-native accent.", "confidence": 0.65, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Sample A has slightly better pacing and expressiveness, though both exhibit a noticeable non-native accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from a strong English accent, but B has slightly better pacing and flow than A, which sounds more disjointed." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly flat delivery.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Disjointed phrasing and unnatural pauses.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Evaluator is not a native speaker of Spanish (Spain).", "Evaluated by a non-native speaker, though the English accent is unmistakable." ] } }, { "id": "f15-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.6241 }, "b": { "src": "audio/f15-fr-FR-B.wav", "model": "s2-pro", "duration": 15.2525 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit a heavy English accent when reading the French text. They are equally intelligible but lack native naturalness.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a heavy English accent when reading the French text. They are equally intelligible but lack native naturalness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from a very strong non-native accent that impacts naturalness and pronunciation. Neither successfully captures a native French delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, e.g., 'tranquille' and 'biscuit'.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'fenêtre' and 'biscuit'.", "severity": "major", "timestamp_seconds": 7.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy non-native accent throughout, similar to Sample A.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'Prends' and 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples feature a very strong non-native (English) accent, making it difficult to assess native French prosody.", "Both samples feature a strong non-native (English) accent, making it difficult to assess native French prosody." ] } }, { "id": "f15-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-de-DE-A.wav", "model": "s2.1-pro", "duration": 17.7139 }, "b": { "src": "audio/f15-de-DE-B.wav", "model": "s2-pro", "duration": 19.2696 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples have a noticeable non-native accent. Sample A is preferred because it has a more natural flow and cadence, whereas Sample B is overly breathy and disjointed.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples have a noticeable non-native accent. Sample A is preferred because it has a more natural flow and cadence, whereas Sample B is overly breathy and disjointed." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B has slightly better prosodic flow and audio quality. Sample A sounds more disjointed and robotic, though both share a noticeable non-native accent." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent on words like 'reserviert' and 'reden'.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, affecting vowels and 'r' sounds.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Overly breathy and disjointed delivery.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat disjointed and unnatural.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'reserviert' and 'genießen'.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native (American) accent in German.", "Both samples exhibit a strong non-native (American) accent, which affects naturalness." ] } }, { "id": "f15-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-pt-BR-A.wav", "model": "s2-pro", "duration": 19.1303 }, "b": { "src": "audio/f15-pt-BR-B.wav", "model": "s2.1-pro", "duration": 18.5266 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples feature a noticeable foreign accent, but Sample A offers slightly smoother pacing and more natural phrasing. Sample B sounds more disjointed and robotic.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples feature a noticeable foreign accent, but Sample A offers slightly smoother pacing and more natural phrasing. Sample B sounds more disjointed and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to meet the locale requirement, using a distinct European Portuguese accent instead of Brazilian Portuguese. They also share a similarly breathy, slightly unnatural delivery, resulting in a tie." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly on 'r' sounds.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The speaker uses a strong European Portuguese accent and vowel reduction, which does not match the requested pt-BR locale.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Choppy and slightly unnatural pacing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "The speaker uses a strong European Portuguese accent and vowel reduction, which does not match the requested pt-BR locale.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong non-native accent for Brazilian Portuguese, but are evaluated on intelligibility and prosody.", "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "f15-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f15-ar-A.wav", "model": "s2-pro", "duration": 21.2433 }, "b": { "src": "audio/f15-ar-B.wav", "model": "s2.1-pro", "duration": 20.6396 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B correctly normalizes and pronounces the time in Arabic. Sample A incorrectly inserts the English word 'four' when reading the time.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B correctly normalizes and pronounces the time in Arabic. Sample A incorrectly inserts the English word 'four' when reading the time." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A correctly reads the time in Arabic, whereas Sample B inserts an English word ('four') when reading the time. Both have noticeable non-native accents, but A is more accurate to the text." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "2.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "wrong_language", "evidence": "Reads '7:45' as 'sab'a four', inserting an English word.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'للاستعجال' as 'listirjal'.", "severity": "minor", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "wrong_language", "evidence": "Reads '7:45' partially in English as 'sab'a four'.", "severity": "major", "timestamp_seconds": 4.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Noticeable non-native accent and robotic prosody throughout.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces 'للاستعجال' as 'listirjal'.", "severity": "minor", "timestamp_seconds": 1.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Non-native pronunciation and stress patterns throughout, such as 'qurb' sounding slightly distorted.", "severity": "minor", "timestamp_seconds": 7.5 } ] }, "limitations": [ "Both models mispronounce 'للاستعجال' slightly, but the primary differentiator is number normalization.", "Both samples exhibit a strong non-native accent in Arabic, making some words sound slightly unnatural." ] } } ] }, { "id": "f16", "name": "girl", "slug": "girl-98d96b", "gender": "Female", "rank": 16, "portrait": "portraits/f16.webp", "language": "8 languages", "popularity": { "callers": 32, "calls": 43 }, "pairs": [ { "id": "f16-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-zh-CN-A.wav", "model": "s2-pro", "duration": 16.042 }, "b": { "src": "audio/f16-zh-CN-B.wav", "model": "s2.1-pro", "duration": 17.621 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more natural conversational volume while maintaining the requested warm and reassuring tone. Sample A is excessively whispery, making it sound less natural for the context.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides a more natural conversational volume while maintaining the requested warm and reassuring tone. Sample A is excessively whispery, making it sound less natural for the context." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A sounds slightly more natural in its phrasing and delivery. Sample B's whisper feels a bit more forced and has slightly more noticeable background noise." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is excessively whispery throughout, lacking the dynamic shift to gentle teasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is heavily whispered, bordering on unnatural.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is heavily whispered rather than just warm and reassuring.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an exaggerated whispered ASMR style which makes naturalness harder to evaluate." ] } }, { "id": "f16-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-ja-JP-A.wav", "model": "s2.1-pro", "duration": 28.07 }, "b": { "src": "audio/f16-ja-JP-B.wav", "model": "s2-pro", "duration": 21.6612 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a much better fit for the requested delivery, despite being somewhat breathy. Sample A sounds like the speaker is crying, which contradicts the 'warm, reassuring' prompt, and includes repeated text at the end.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B provides a much better fit for the requested delivery, despite being somewhat breathy. Sample A sounds like the speaker is crying, which contradicts the 'warm, reassuring' prompt, and includes repeated text at the end." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a much more natural and appropriate delivery for the requested warm and reassuring tone. Sample B includes excessive, unnatural breathing and gasping sounds that make the speaker sound distressed rather than reassuring." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker sounds highly distressed and crying, contradicting the 'warm, reassuring' and 'gently teasing' directions.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "text_addition", "evidence": "Adds an extra 'でも' (demo) and breathing sounds at the end of the recording.", "severity": "major", "timestamp_seconds": 26.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Excessive gasping and heavy breathing throughout, sounding distressed rather than warm and reassuring.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Added 'でも' at the very end of the recording.", "severity": "minor", "timestamp_seconds": 26.5 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is extremely breathy and whispered, which is slightly unnatural for a normal conversation.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "other", "evidence": "Slightly breathy delivery, but generally acceptable for the intended tone.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Japanese speaker, but can evaluate basic pronunciation and text fidelity.", "I am not a native Japanese speaker, but I can evaluate the general prosody, intelligibility, and text fidelity." ] } }, { "id": "f16-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-ko-KR-A.wav", "model": "s2-pro", "duration": 19.1071 }, "b": { "src": "audio/f16-ko-KR-B.wav", "model": "s2.1-pro", "duration": 20.8718 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural phrasing and pacing. Sample B feels a bit more stilted, though both rely heavily on an exaggerated breathy delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly more natural phrasing and pacing. Sample B feels a bit more stilted, though both rely heavily on an exaggerated breathy delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery that better matches the requested warm and gently teasing tone. Sample A sounds slightly more robotic and flat." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is extremely breathy, sounding more like ASMR than a standard warm conversation.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice is very breathy and whispered, which feels exaggerated.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and phrasing feel slightly stilted and less fluid than ideal.", "severity": "minor", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested teasing tone at the end.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an exaggerated breathy tone that borders on ASMR, which may not perfectly align with standard conversational expectations.", "I am not a native Korean speaker, so subtle accent nuances may be missed." ] } }, { "id": "f16-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-es-ES-A.wav", "model": "s2-pro", "duration": 18.5034 }, "b": { "src": "audio/f16-es-ES-B.wav", "model": "s2.1-pro", "duration": 18.3176 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A is preferred for its clearer pronunciation and better intelligibility. Sample B suffers from slurred speech and a noticeable non-native accent.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A is preferred for its clearer pronunciation and better intelligibility. Sample B suffers from slurred speech and a noticeable non-native accent." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B is preferred because it has clearer articulation and less background noise, making the heavily whispered delivery more intelligible than Sample A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 2.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper (ASMR style) rather than a normal warm conversation.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than just warm and reassuring.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Slurred and unclear pronunciation on words like 'junto'.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly whispered delivery.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss and slight distortion throughout the whispered speech.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an extreme ASMR/whispered style which makes evaluating normal prosody difficult.", "Both samples use an extreme whispered delivery that borders on ASMR, which may not be universally suitable." ] } }, { "id": "f16-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-fr-FR-A.wav", "model": "s2-pro", "duration": 15.5544 }, "b": { "src": "audio/f16-fr-FR-B.wav", "model": "s2.1-pro", "duration": 17.2263 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a much more natural and appropriate delivery, capturing the warmth and teasing effectively. Sample A relies on an excessive, unnatural whisper that reduces intelligibility and audio quality.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B offers a much more natural and appropriate delivery, capturing the warmth and teasing effectively. Sample A relies on an excessive, unnatural whisper that reduces intelligibility and audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples adopt an overly breathy, whispering delivery. Sample B is slightly preferred as it has fewer synthetic artifacts in the whisper compared to Sample A." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is an excessive whisper rather than a natural warm tone.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than a standard warm conversational tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "The whisper effect sounds slightly synthetic and introduces minor artifacts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than a standard warm conversational tone.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an extreme ASMR-style whisper which may not perfectly align with standard conversational expectations." ] } }, { "id": "f16-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-de-DE-A.wav", "model": "s2-pro", "duration": 18.573 }, "b": { "src": "audio/f16-de-DE-B.wav", "model": "s2.1-pro", "duration": 20.4538 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A has more natural phrasing and cadence despite the heavy whispering. Sample B suffers from awkward pauses and a slightly unnatural rhythm.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A has more natural phrasing and cadence despite the heavy whispering. Sample B suffers from awkward pauses and a slightly unnatural rhythm." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples use an overly breathy, whispering delivery that sounds somewhat unnatural. Sample B is slightly preferred due to more natural German pronunciation, whereas A has a slight non-native accent." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is an exaggerated whisper rather than a normal warm tone.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is excessively breathy and whispering, which feels exaggerated for the prompt.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Awkward pauses and unnatural cadence throughout the sentence.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and whispery delivery.", "severity": "minor", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Slight non-native accent on words like 'reserviert' and 'Keks'.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is excessively breathy, sounding more like a heavy whisper than warm speech.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an exaggerated ASMR-style whisper, making it difficult to assess normal phonation." ] } }, { "id": "f16-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-pt-BR-A.wav", "model": "s2.1-pro", "duration": 18.5266 }, "b": { "src": "audio/f16-pt-BR-B.wav", "model": "s2-pro", "duration": 21.3594 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with a very similar, highly breathy vocal style. Neither clearly distinguishes the 'gently teasing' shift, resulting in a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with a very similar, highly breathy vocal style. Neither clearly distinguishes the 'gently teasing' shift, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural and conversational delivery that fits the directions well. Sample A relies on an overly breathy, whispered tone that feels unnatural for the context." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3, "expressiveness": 3, "audioQuality": 3 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery remains uniformly breathy and does not clearly shift to a teasing tone at the end.", "severity": "minor", "timestamp_seconds": 14.5 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice maintains a whispered, breathy tone throughout, missing the shift to gentle teasing.", "severity": "minor", "timestamp_seconds": 16.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery is entirely whispered, which feels overly intimate and unnatural for the context.", "severity": "major", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples use an extremely breathy, ASMR-like delivery which may not be universally preferred for 'warm and reassuring'.", "I am not a native speaker of Brazilian Portuguese, but I can assess general pronunciation and prosody." ] } }, { "id": "f16-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f16-ar-A.wav", "model": "s2.1-pro", "duration": 22.5204 }, "b": { "src": "audio/f16-ar-B.wav", "model": "s2-pro", "duration": 22.4043 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples adopt an overly whispered delivery that harms intelligibility. Sample A is slightly louder and clearer than Sample B, which is too muffled.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples adopt an overly whispered delivery that harms intelligibility. Sample A is slightly louder and clearer than Sample B, which is too muffled." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit inappropriate whispering throughout, making them difficult to hear and failing the requested delivery." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 2.5 }, "B": { "naturalness": 2.5, "intelligibility": 2, "expressiveness": 2.5, "audioQuality": 2 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is an extreme whisper rather than just warm and reassuring.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The entire sample is whispered rather than spoken warmly.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "other", "evidence": "Overall volume is extremely low.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice is heavily whispered, misinterpreting the warm direction.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "distortion", "evidence": "The extreme whispering causes the audio to sound muffled and unclear.", "severity": "major", "timestamp_seconds": 5.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The entire sample is whispered rather than spoken warmly.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "other", "evidence": "Overall volume is extremely low.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use an extreme whispered ASMR style, making standard intelligibility and pronunciation evaluation difficult.", "Arabic proficiency is limited; evaluation focuses on general audio characteristics and basic word matching." ] } } ] }, { "id": "f17", "name": "Remy", "slug": "remy", "gender": "Female", "rank": 17, "portrait": "portraits/f17.webp", "language": "8 languages", "popularity": { "callers": 26, "calls": 39 }, "pairs": [ { "id": "f17-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-zh-CN-A.wav", "model": "s2-pro", "duration": 17.2959 }, "b": { "src": "audio/f17-zh-CN-B.wav", "model": "s2.1-pro", "duration": 17.203 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B features much more natural prosody and native-like pronunciation. Sample A has a noticeable non-native accent and somewhat robotic delivery.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample B features much more natural prosody and native-like pronunciation. Sample A has a noticeable non-native accent and somewhat robotic delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A features a natural, native-sounding Mandarin pronunciation with appropriate prosody and tone. Sample B has a noticeable foreign accent that negatively impacts the naturalness and tonal accuracy of the speech." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Flat and somewhat robotic cadence.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent affecting tones and prosody throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [] }, "limitations": [] } }, { "id": "f17-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-ja-JP-A.wav", "model": "s2.1-pro", "duration": 20.5003 }, "b": { "src": "audio/f17-ja-JP-B.wav", "model": "s2-pro", "duration": 19.8733 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples share a major pronunciation error, reading '明日' as 'asa'. However, Sample A is preferred for its significantly better expressiveness and naturalness, whereas Sample B sounds flat and robotic.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples share a major pronunciation error, reading '明日' as 'asa'. However, Sample A is preferred for its significantly better expressiveness and naturalness, whereas Sample B sounds flat and robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples share a major pronunciation error, reading 'ashita' as 'asa'. Sample A is preferred for its slightly better pacing and warmer expressiveness." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日' as 'asa' instead of 'ashita' or 'asu'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Misreads '明日' (ashita) as 'asa', saying 'asa no yoru'.", "severity": "major", "timestamp_seconds": 2.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日' as 'asa' instead of 'ashita' or 'asu'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat and lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Misreads '明日' (ashita) as 'asa', saying 'asa no yoru'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both models mispronounce '明日' (ashita/asu) as 'asa' (morning).", "Both models misread the kanji for 'tomorrow' (明日) as 'morning' (朝), resulting in the nonsensical phrase 'morning's night'." ] } }, { "id": "f17-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-ko-KR-A.wav", "model": "s2.1-pro", "duration": 20.4074 }, "b": { "src": "audio/f17-ko-KR-B.wav", "model": "s2-pro", "duration": 19.4322 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A delivers a much more natural and expressive performance, capturing the requested warmth. Sample B sounds robotic and flat.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A delivers a much more natural and expressive performance, capturing the requested warmth. Sample B sounds robotic and flat." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B provides much more natural prosody and better captures the requested warm and teasing tones. Sample A sounds flat and robotic throughout." } ], "scores": { "A": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 2, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The cadence is stiff and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "The tone is flat and misses the requested warmth and teasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks natural cadence or emotional variation.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm, reassuring, or gently teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f17-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-es-ES-A.wav", "model": "s2-pro", "duration": 17.6442 }, "b": { "src": "audio/f17-es-ES-B.wav", "model": "s2.1-pro", "duration": 19.1535 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from a strong foreign accent rather than a native Spanish pronunciation. Sample B is preferred as it has slightly more fluid phrasing, whereas Sample A sounds more robotic.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a strong foreign accent rather than a native Spanish pronunciation. Sample B is preferred as it has slightly more fluid phrasing, whereas Sample A sounds more robotic." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples exhibit a very strong, non-native English accent when speaking Spanish. Sample A is slightly preferred as its prosody and phrasing are marginally less robotic than Sample B." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong English accent throughout the recording, affecting vowels and consonants.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is choppy and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Phrasing is choppy and unnatural.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable English accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native English accent throughout the Spanish text.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a strong English accent, making it difficult to assess native Spanish (Spain) prosody." ] } }, { "id": "f17-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-fr-FR-A.wav", "model": "s2.1-pro", "duration": 16.9708 }, "b": { "src": "audio/f17-fr-FR-B.wav", "model": "s2-pro", "duration": 17.203 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Both samples suffer from severe pronunciation issues, sounding like an English speaker reading French phonetically. Sample A is slightly preferred for having marginally better prosody and expressiveness compared to B's robotic delivery.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Both samples suffer from severe pronunciation issues, sounding like an English speaker reading French phonetically. Sample A is slightly preferred for having marginally better prosody and expressiveness compared to B's robotic delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples have a heavy foreign accent. Sample B is slightly preferred as its pronunciation is marginally clearer than A." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'Prends ton temps', 'réservé', 'tranquille'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Strong non-native accent throughout the sample.", "severity": "major", "timestamp_seconds": 1.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent, e.g., 'réservé' sounds like 'herzervey', 'fenêtre' like 'fenate'.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is flat and disjointed.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Heavy non-native accent distorts words like 'réservé' and 'fenêtre'.", "severity": "major", "timestamp_seconds": 1.5 } ] }, "limitations": [ "Both samples exhibit a very strong English accent, making the French pronunciation highly unnatural.", "Both samples feature a very strong non-native (American) accent, affecting pronunciation." ] } }, { "id": "f17-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-de-DE-A.wav", "model": "s2-pro", "duration": 19.5018 }, "b": { "src": "audio/f17-de-DE-B.wav", "model": "s2.1-pro", "duration": 20.0359 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from a severe foreign accent and robotic prosody. Neither successfully conveys the requested warm or teasing delivery, resulting in a tie.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe foreign accent and robotic prosody. Neither successfully conveys the requested warm or teasing delivery, resulting in a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a heavy non-native accent and robotic prosody. Sample A is preferred because its pronunciation is slightly more accurate, whereas Sample B mispronounces 'morgen' as 'morging'." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 2.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy American accent throughout, mispronouncing vowels and consonants (e.g., 'Lass dir Zeit', 'reserviert').", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Flat, disjointed cadence lacking natural German intonation.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to sound warm, reassuring, or gently teasing.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'morgen' as 'morging'.", "severity": "major", "timestamp_seconds": 2.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 17.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is disjointed and unnatural.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy American accent throughout, mispronouncing vowels and consonants (e.g., 'Lass dir Zeit', 'reserviert').", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "robotic_prosody", "evidence": "Flat, disjointed cadence lacking natural German intonation.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Fails to sound warm, reassuring, or gently teasing.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'Keks' like the English word 'cakes'.", "severity": "major", "timestamp_seconds": 18.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is stiff and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a very strong foreign (American) accent, making native-level evaluation of subtle prosody difficult.", "Both samples feature a very strong non-native (American) accent that affects the pronunciation of several German words." ] } }, { "id": "f17-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-pt-BR-A.wav", "model": "s2.1-pro", "duration": 20.7325 }, "b": { "src": "audio/f17-pt-BR-B.wav", "model": "s2-pro", "duration": 18.6659 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples have noticeable non-native accents and struggle with natural Brazilian Portuguese prosody. They are equally intelligible but lack authentic regional pronunciation.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples have noticeable non-native accents and struggle with natural Brazilian Portuguese prosody. They are equally intelligible but lack authentic regional pronunciation." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.9, "rationale": "Both samples suffer from a severe foreign accent that degrades naturalness and pronunciation in Brazilian Portuguese. Neither is suitable for a native-sounding application." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native (Americanized) accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent affects words like 'amanhã' and 'tranquilo'.", "severity": "major", "timestamp_seconds": 4.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Pronunciation leans towards European Portuguese rather than the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Strong foreign accent affects words like 'amanhã' and 'pressa'.", "severity": "major", "timestamp_seconds": 3.5 } ] }, "limitations": [ "Both samples exhibit strong non-native accents for Brazilian Portuguese.", "Both samples exhibit a strong non-native accent, making it difficult to assess native-level prosody." ] } }, { "id": "f17-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f17-ar-A.wav", "model": "s2-pro", "duration": 21.3826 }, "b": { "src": "audio/f17-ar-B.wav", "model": "s2.1-pro", "duration": 20.2216 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples exhibit similar robotic prosody and unnatural pauses, particularly before 'النافذة'. They both convey the text accurately but lack natural conversational flow.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar robotic prosody and unnatural pauses, particularly before 'النافذة'. They both convey the text accurately but lack natural conversational flow." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples are highly similar in voice quality, pacing, and pronunciation. They both feature an unnatural pause before 'an-nafidha' and have a slightly robotic prosody. Neither stands out as significantly better." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة' and generally stilted delivery.", "severity": "major", "timestamp_seconds": 8.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafidha'.", "severity": "minor", "timestamp_seconds": 8.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause before 'النافذة' and generally stilted delivery.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafidha'.", "severity": "minor", "timestamp_seconds": 8.0 } ] }, "limitations": [ "Evaluation of Arabic prosody and naturalness is based on general MSA standards.", "Both samples exhibit a noticeable non-native or synthetic accent in Arabic.", "Evaluation of exact vowel endings (tashkeel) is lenient as both use a mix of formal and colloquial pausing." ] } } ] }, { "id": "f18", "name": "Amanda", "slug": "amanda-d62308", "gender": "Female", "rank": 18, "portrait": "portraits/f18.webp", "language": "8 languages", "popularity": { "callers": 26, "calls": 37 }, "pairs": [ { "id": "f18-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-zh-CN-A.wav", "model": "s2-pro", "duration": 16.1349 }, "b": { "src": "audio/f18-zh-CN-B.wav", "model": "s2.1-pro", "duration": 16.7619 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural phrasing and better captures the warm, reassuring tone requested. Sample A is clear but sounds a bit more robotic in its delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers slightly more natural phrasing and better captures the warm, reassuring tone requested. Sample A is clear but sounds a bit more robotic in its delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more natural and expressive delivery, better capturing the requested warm and teasing tones. Sample B is slightly flatter in comparison." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat delivery lacking the requested warmth and teasing contrast.", "severity": "minor", "timestamp_seconds": null } ], "B": [] }, "limitations": [] } }, { "id": "f18-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-ja-JP-A.wav", "model": "s2-pro", "duration": 18.8052 }, "b": { "src": "audio/f18-ja-JP-B.wav", "model": "s2.1-pro", "duration": 19.6876 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A reads the text accurately, whereas Sample B mispronounces a key word, changing the meaning. Both samples are somewhat stiff in delivery.", "confidence": 0.9, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample A reads the text accurately, whereas Sample B mispronounces a key word, changing the meaning. Both samples are somewhat stiff in delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.9, "rationale": "Sample B delivers a more natural and expressive performance that fits the requested tone. Sample A mispronounces 'ashita' as 'asa' and sounds somewhat robotic." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "3.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is somewhat stiff and lacks the requested warmth and teasing tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Mispronounces '明日' (ashita) as 'asa' (morning), resulting in the contradictory phrase 'asa no yoru'.", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat unnatural and flat.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Mispronounces 'ashita' (tomorrow) as 'asa' (morning).", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f18-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-ko-KR-A.wav", "model": "s2-pro", "duration": 18.782 }, "b": { "src": "audio/f18-ko-KR-B.wav", "model": "s2.1-pro", "duration": 18.3176 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers slightly more natural phrasing and better captures the intended warmth and teasing tone compared to Sample A, which sounds a bit stiffer.", "confidence": 0.65, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample B offers slightly more natural phrasing and better captures the intended warmth and teasing tone compared to Sample A, which sounds a bit stiffer." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.6, "rationale": "Sample A offers slightly more natural phrasing and pacing. Sample B is accurate but sounds a bit more deliberate and robotic in its cadence." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly stiff cadence in the middle sentences.", "severity": "minor", "timestamp_seconds": 7.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is somewhat slow and deliberate, lacking natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing throughout the utterance.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Non-native speaker evaluating subtle emotional shifts in Korean.", "I am not a native Korean speaker, but can assess general pronunciation and prosody." ] } }, { "id": "f18-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-es-ES-A.wav", "model": "s2.1-pro", "duration": 17.6674 }, "b": { "src": "audio/f18-es-ES-B.wav", "model": "s2-pro", "duration": 18.2479 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Both samples suffer from a severe English accent when speaking Spanish. Sample B is preferred for having slightly smoother prosody and phrasing compared to the disjointed delivery in Sample A.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples suffer from a severe English accent when speaking Spanish. Sample B is preferred for having slightly smoother prosody and phrasing compared to the disjointed delivery in Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a more natural and expressive delivery, capturing the warm and reassuring tone well. Sample B sounds more robotic and lacks the requested emotional shifts." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the entire Spanish text.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Disjointed and unnatural phrasing, lacking conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Severe English accent throughout the entire Spanish text.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a severe foreign accent, making it difficult to assess native-level nuances." ] } }, { "id": "f18-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.7866 }, "b": { "src": "audio/f18-fr-FR-B.wav", "model": "s2-pro", "duration": 15.508 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have a noticeable non-native accent, but Sample B offers slightly smoother prosody and better pacing than Sample A.", "confidence": 0.65, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Both samples have a noticeable non-native accent, but Sample B offers slightly smoother prosody and better pacing than Sample A." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.6, "rationale": "Both samples suffer from robotic prosody and unnatural intonation. Sample B is marginally preferred for having slightly better expressiveness that aligns closer to the requested delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Strong non-native accent throughout, particularly noticeable on 'Prends ton temps'.", "severity": "major", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is stiff and unnatural for conversational French.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Noticeable non-native accent throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is flat and lacks natural phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Fails to convey the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a strong non-native accent that affects the naturalness of the French pronunciation.", "Both samples exhibit significant non-native or robotic prosody." ] } }, { "id": "f18-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-de-DE-A.wav", "model": "s2-pro", "duration": 18.782 }, "b": { "src": "audio/f18-de-DE-B.wav", "model": "s2.1-pro", "duration": 18.2944 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples have a similar level of quality and text fidelity. Both exhibit a noticeable English accent, especially on the word 'Keks', making it a tie.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples have a similar level of quality and text fidelity. Both exhibit a noticeable English accent, especially on the word 'Keks', making it a tie." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers slightly better naturalness and expressiveness. Sample B sounds more robotic and has a slight non-native accent." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The word 'Keks' is pronounced with an English accent, sounding like 'cakes'.", "severity": "minor", "timestamp_seconds": 16.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The cadence is robotic and lacks natural flow.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Slight non-native accent on words like 'reserviert'.", "severity": "minor", "timestamp_seconds": 6.0 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "The word 'Keks' is pronounced with an English accent, sounding like 'cakes'.", "severity": "minor", "timestamp_seconds": 16.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warm and teasing tones.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, particularly on the word 'Keks'." ] } }, { "id": "f18-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-pt-BR-A.wav", "model": "s2-pro", "duration": 19.316 }, "b": { "src": "audio/f18-pt-BR-B.wav", "model": "s2.1-pro", "duration": 19.525 }, "judgment": { "winner": "tie", "reason": "Both orders judge a tie. First-order review: Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese pronunciation. They are otherwise comparable in intelligibility and audio quality.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese locale, defaulting to European Portuguese pronunciation. They are otherwise comparable in intelligibility and audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text clearly but fail to use the requested Brazilian Portuguese accent, sounding distinctly European Portuguese instead. They are tied in overall quality." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "The accent is distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 17.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The pronunciation and intonation are strongly European Portuguese, missing the pt-BR locale.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "The accent is distinctly European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "text_substitution", "evidence": "Reads 'pra mim' as 'para mim'.", "severity": "minor", "timestamp_seconds": 17.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The accent sounds like European Portuguese, not the requested Brazilian Portuguese.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples use a European Portuguese accent instead of the requested Brazilian Portuguese locale.", "Both samples exhibit a strong European Portuguese accent rather than the requested Brazilian Portuguese locale." ] } }, { "id": "f18-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f18-ar-A.wav", "model": "s2.1-pro", "duration": 20.1984 }, "b": { "src": "audio/f18-ar-B.wav", "model": "s2-pro", "duration": 21.1969 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is preferred because it maintains better text fidelity. Sample A inserts an extra syllable or word before 'an-nafitha', whereas B only has an unnatural pause.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is preferred because it maintains better text fidelity. Sample A inserts an extra syllable or word before 'an-nafitha', whereas B only has an unnatural pause." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides better text fidelity and pronunciation. Sample B includes an inserted syllable and a grammatical pronunciation error." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "3.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_addition", "evidence": "Adds an extra syllable or word (sounds like 'hum') between 'qurb' and 'an-nafitha'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and intonation are disjointed and unnatural throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_addition", "evidence": "Inserted an extra syllable or word before 'النافذة'.", "severity": "major", "timestamp_seconds": 7.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounced 'هادئة' incorrectly as 'هادئا'.", "severity": "minor", "timestamp_seconds": 12.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pause between 'qurb' and 'an-nafitha', and generally stilted delivery.", "severity": "major", "timestamp_seconds": 8.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and phrasing throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit stilted, somewhat robotic prosody, making naturalness difficult to rate highly." ] } } ] }, { "id": "f19", "name": "Sahithi", "slug": "sahithi-17e3f8", "gender": "Female", "rank": 19, "portrait": "portraits/f19.webp", "language": "8 languages", "popularity": { "callers": 24, "calls": 34 }, "pairs": [ { "id": "f19-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.2975 }, "b": { "src": "audio/f19-zh-CN-B.wav", "model": "s2-pro", "duration": 15.4383 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A offers slightly better pacing and a more natural delivery. Sample B sounds more robotic and flat. Both samples suffer from muffled audio quality.", "confidence": 0.85, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A offers slightly better pacing and a more natural delivery. Sample B sounds more robotic and flat. Both samples suffer from muffled audio quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a more natural and expressive delivery that better matches the requested tone. Sample B suffers from noticeable background noise and a slightly robotic cadence." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Pacing is somewhat stiff and lacks dynamic range.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Audio sounds slightly muffled with some background artifacts.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background noise and metallic tone throughout.", "severity": "minor", "timestamp_seconds": 0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly flat and robotic delivery, lacking the requested warmth.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Delivery is flat, disjointed, and lacks the requested emotional shifts.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Audio is muffled and sounds heavily processed.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [] } }, { "id": "f19-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.316 }, "b": { "src": "audio/f19-ja-JP-B.wav", "model": "s2-pro", "duration": 18.1086 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a smoother, more natural prosody and cleaner audio quality. Sample A has noticeable background noise and slightly choppy delivery.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a smoother, more natural prosody and cleaner audio quality. Sample A has noticeable background noise and slightly choppy delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A has slightly more natural pitch accents and phrasing than Sample B, which suffers from highly unnatural intonation on several words." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background noise and hissing throughout the recording.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly choppy phrasing and unnatural pauses.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Highly unnatural pitch accents on words like 'hanasou' and 'yoyaku'.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is very breathy, bordering on a whisper, which may be overly stylized.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The overall intonation is somewhat flat and lacks the natural cadence of native Japanese speech.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle emotional shifts in Japanese may be limited by non-native proficiency.", "Both samples exhibit noticeable background hiss and somewhat robotic prosody." ] } }, { "id": "f19-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-ko-KR-A.wav", "model": "s2-pro", "duration": 17.7371 }, "b": { "src": "audio/f19-ko-KR-B.wav", "model": "s2.1-pro", "duration": 18.0158 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides a more natural and appropriate delivery. Sample A relies on an exaggerated whisper that sounds unnatural and introduces breathy noise.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides a more natural and appropriate delivery. Sample A relies on an exaggerated whisper that sounds unnatural and introduces breathy noise." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more appropriate delivery for the context. Sample B uses an overly breathy, whispered tone that does not fit the requested warm and teasing directions." } ], "scores": { "A": { "naturalness": 2.5, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The speaker whispers the entire passage, which does not fit the requested warm and reassuring tone.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and whispered, which does not fit the warm and teasing directions.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly stiff pacing and intonation throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "Lacks a clear shift to gentle teasing at the end.", "severity": "minor", "timestamp_seconds": 12.5 } ] }, "limitations": [ "I am not a native Korean speaker, but can assess general pronunciation and prosody.", "I am not a native Korean speaker, so subtle accent nuances may be missed." ] } }, { "id": "f19-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-es-ES-A.wav", "model": "s2-pro", "duration": 15.3919 }, "b": { "src": "audio/f19-es-ES-B.wav", "model": "s2.1-pro", "duration": 16.669 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B provides clearer audio and better intelligibility. Sample A is overly breathy and muffled, resulting in an unnatural, whisper-like delivery that detracts from the intended warmth.", "confidence": 0.8, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B provides clearer audio and better intelligibility. Sample A is overly breathy and muffled, resulting in an unnatural, whisper-like delivery that detracts from the intended warmth." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is more natural and intelligible, though both struggle with a native Spanish accent. Sample B is excessively breathy and fails to capture the intended delivery." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The voice is overly breathy and whisper-like rather than warm and reassuring.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing and cadence feel unnatural and disjointed.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is excessively breathy and whispering, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'He' incorrectly.", "severity": "minor", "timestamp_seconds": 1.5 } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The speech sounds somewhat mechanical with flat intonation.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested gentle teasing at the end.", "severity": "minor", "timestamp_seconds": 13.0 }, { "order": "BA", "category": "pronunciation", "evidence": "Pronounces 'He' with a harsh 'j' sound.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing and intonation sound slightly synthetic and disjointed.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Non-native speaker evaluation of es-ES accent nuances." ] } }, { "id": "f19-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.717 }, "b": { "src": "audio/f19-fr-FR-B.wav", "model": "s2-pro", "duration": 14.4631 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples suffer from severe synthetic artifacts and unnatural delivery. Sample A is robotic and choppy, while Sample B is overly breathy and whispered. Neither provides a high-quality listening experience.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples suffer from severe synthetic artifacts and unnatural delivery. Sample A is robotic and choppy, while Sample B is overly breathy and whispered. Neither provides a high-quality listening experience." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A is preferred for its more natural phrasing and warmer tone, despite being overly breathy. Sample B suffers from noticeably robotic and disjointed prosody." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The overall cadence is choppy and lacks natural human phrasing.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Noticeable synthetic buzz and artifacts throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation are disjointed and synthetic throughout the sample.", "severity": "major", "timestamp_seconds": 2.0 } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The entire sample is whispered, which is an exaggerated interpretation of 'warm, reassuring'.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Breathy artifacts and synthetic noise are present throughout.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The delivery remains uniformly breathy and subdued, missing the shift to gentle teasing.", "severity": "minor", "timestamp_seconds": 11.0 } ] }, "limitations": [ "Both samples exhibit significant synthetic artifacts and unnatural prosody.", "Evaluation of subtle regional French accents is limited." ] } }, { "id": "f19-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-de-DE-A.wav", "model": "s2-pro", "duration": 17.6907 }, "b": { "src": "audio/f19-de-DE-B.wav", "model": "s2.1-pro", "duration": 19.2928 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides better naturalness and expressiveness than Sample B, which sounds highly robotic and disjointed. Both samples suffer from muffled audio quality, but A is more usable overall.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides better naturalness and expressiveness than Sample B, which sounds highly robotic and disjointed. Both samples suffer from muffled audio quality, but A is more usable overall." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A provides a more standard conversational delivery, though slightly robotic. Sample B is entirely whispered, which does not fit the requested delivery." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 2.5, "audioQuality": 3 }, "B": { "naturalness": 2.5, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and pacing throughout.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Muffled audio quality with some background hiss.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The entire sample is whispered, missing the warm and teasing cues.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Highly disjointed and robotic delivery.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Lacks the requested warm and teasing tones.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "Muffled and synthetic-sounding audio.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested emotional shifts.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit muffled audio quality and synthetic artifacts.", "Evaluator is not a native German speaker, but can assess general prosody and text fidelity." ] } }, { "id": "f19-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-pt-BR-A.wav", "model": "s2.1-pro", "duration": 18.6427 }, "b": { "src": "audio/f19-pt-BR-B.wav", "model": "s2-pro", "duration": 18.1318 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B is slightly more natural and expressive, despite the heavy whispering. Sample A suffers from severe robotic prosody and choppy delivery.", "confidence": 0.85, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B is slightly more natural and expressive, despite the heavy whispering. Sample A suffers from severe robotic prosody and choppy delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.9, "rationale": "Sample A provides a much clearer and more natural delivery that fits the requested warm tone. Sample B suffers from significant audio degradation and muffling, making it sound robotic and less pleasant." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3.5, "expressiveness": 2.5, "audioQuality": 2 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 } }, "fidelity": { "A": "5.0/5 mean text-fidelity score", "B": "5.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The delivery is highly disjointed and robotic throughout the sample.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "distortion", "evidence": "The audio sounds muffled and heavily synthetic.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Severe muffling and low-bitrate artifacts throughout the recording.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The delivery is somewhat flat and lacks the requested warmth.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The entire sample is whispered, which is an extreme interpretation of the warm direction.", "severity": "minor", "timestamp_seconds": null }, { "order": "AB", "category": "robotic_prosody", "evidence": "Some phrasing remains slightly unnatural and stiff.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Slight background hiss and muffled quality throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit synthetic artifacts and unnatural prosody." ] } }, { "id": "f19-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f19-ar-A.wav", "model": "s2.1-pro", "duration": 18.8981 }, "b": { "src": "audio/f19-ar-B.wav", "model": "s2-pro", "duration": 19.0142 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample B offers a slightly more natural and expressive delivery, capturing the warm and reassuring tone better than Sample A. Both samples have minor text fidelity issues, but B's overall prosody is more pleasing.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a slightly more natural and expressive delivery, capturing the warm and reassuring tone better than Sample A. Both samples have minor text fidelity issues, but B's overall prosody is more pleasing." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B offers clearer articulation and better audio quality. Sample A is excessively breathy and whispering, which detracts from naturalness and intelligibility." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 3.5, "expressiveness": 3.5, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "text_substitution", "evidence": "Heard 'للاستعجال' instead of 'للاستعجال'.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "AB", "category": "pronunciation", "evidence": "Slight mispronunciation of 'بسكويت'.", "severity": "minor", "timestamp_seconds": 16.0 }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing and intonation are somewhat stiff and lack dynamic range.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "text_substitution", "evidence": "Heard 'للاستعجال' instead of 'للاستعجال'.", "severity": "minor", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The voice is excessively breathy and whispering, which sounds unnatural rather than warm.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native speaker of Modern Standard Arabic, so my evaluation of subtle pronunciation nuances may be limited.", "I am not a native speaker of Modern Standard Arabic, but can evaluate general intelligibility, text fidelity, and audio quality." ] } } ] }, { "id": "f20", "name": "Kinsley Zimmerman.", "slug": "kinsley-zimmerman-ae675b", "gender": "Female", "rank": 20, "portrait": "portraits/f20.webp", "language": "8 languages", "popularity": { "callers": 23, "calls": 31 }, "pairs": [ { "id": "f20-zh-CN", "scenario": "Multilingual conversation", "scenarioId": "zh-CN", "languageId": "zh-CN", "languageCode": "zh-CN", "locale": "zh-CN", "text": "[warm, reassuring] 慢慢来，不着急。我订好了明天晚上7点45分的两个座位。我们可以坐在窗边，聊聊你今天过得怎么样，一起享受片刻的安静。[gently teasing] 不过先说好，这次最后一块饼干可要留给我。", "language": "Mandarin Chinese (Simplified Chinese)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-zh-CN-A.wav", "model": "s2.1-pro", "duration": 16.3439 }, "b": { "src": "audio/f20-zh-CN-B.wav", "model": "s2-pro", "duration": 16.994 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Sample A provides a more natural conversational tone that fits the requested warmth and teasing. Sample B is overly breathy and whispered, which feels unnatural for the context.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample A provides a more natural conversational tone that fits the requested warmth and teasing. Sample B is overly breathy and whispered, which feels unnatural for the context." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a more natural pacing that better suits the warm and reassuring delivery direction. Sample B feels slightly rushed in comparison." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "BA", "category": "delivery_mismatch", "evidence": "The pacing is slightly fast for a warm and reassuring tone.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "Overly breathy and whispered delivery throughout.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluation of subtle regional accent variations is limited." ] } }, { "id": "f20-ja-JP", "scenario": "Multilingual conversation", "scenarioId": "ja-JP", "languageId": "ja-JP", "languageCode": "ja-JP", "locale": "ja-JP", "text": "[warm, reassuring] 急がなくていいよ。明日の夜7時45分に、二人分の席を予約しておいたんだ。窓際に座って、今日どんな一日だったか話そう。二人でゆっくり過ごせたらいいね。[gently teasing] でも、約束だよ。今度は最後のクッキーを残しておいてね。", "language": "Japanese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-ja-JP-A.wav", "model": "s2.1-pro", "duration": 19.943 }, "b": { "src": "audio/f20-ja-JP-B.wav", "model": "s2-pro", "duration": 20.3842 }, "judgment": { "winner": "B", "reason": "Both orders prefer gallery B. First-order review: Sample B offers a more natural and expressive delivery, capturing the warm and reassuring tone well. Sample A suffers from noticeable audio quality issues, including a persistent hiss and some distortion.", "confidence": 0.825, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "B", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample B offers a more natural and expressive delivery, capturing the warm and reassuring tone well. Sample A suffers from noticeable audio quality issues, including a persistent hiss and some distortion." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.85, "rationale": "Sample A better captures the requested warm and reassuring tone, whereas Sample B sounds overly breathless and giggly throughout the entire recording." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3 }, "B": { "naturalness": 4, "intelligibility": 4, "expressiveness": 4, "audioQuality": 4 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "distortion", "evidence": "Noticeable background hiss and slight distortion throughout the recording.", "severity": "major", "timestamp_seconds": 0.0 }, { "order": "BA", "category": "delivery_mismatch", "evidence": "The speaker sounds breathless and giggly from the beginning, which does not fit the 'warm, reassuring' prompt.", "severity": "minor", "timestamp_seconds": 0.0 } ], "B": [] }, "limitations": [] } }, { "id": "f20-ko-KR", "scenario": "Multilingual conversation", "scenarioId": "ko-KR", "languageId": "ko-KR", "languageCode": "ko-KR", "locale": "ko-KR", "text": "[warm, reassuring] 천천히 와도 괜찮아. 내일 저녁 7시 45분에 두 자리 예약해 뒀어. 창가에 앉아서 오늘 하루 어땠는지 이야기하자. 우리 둘이 조용히 쉬는 시간을 가져도 좋겠지. [gently teasing] 대신 약속 하나만 해 줘. 이번에는 마지막 쿠키를 내 몫으로 남겨 두기야.", "language": "Korean", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-ko-KR-A.wav", "model": "s2-pro", "duration": 18.5266 }, "b": { "src": "audio/f20-ko-KR-B.wav", "model": "s2.1-pro", "duration": 17.5281 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples are intelligible and follow the text well. Sample A adopts a highly breathy, whispered tone, while Sample B is clearer but slightly rushed and less expressive. Neither perfectly captures the requested delivery shifts.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.7, "rationale": "Both samples are intelligible and follow the text well. Sample A adopts a highly breathy, whispered tone, while Sample B is clearer but slightly rushed and less expressive. Neither perfectly captures the requested delivery shifts." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Sample A offers a more natural and expressive delivery that better captures the requested warm and teasing tones. Sample B is overly breathy and has slightly stilted pacing." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 4 }, "B": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "delivery_mismatch", "evidence": "The delivery is entirely whispered and breathy, which does not fully align with 'warm, reassuring' and 'gently teasing'.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Pacing is slightly stilted and overly breathy.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "The pacing is slightly rushed and lacks the warmth and teasing nuances requested.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "I am not a native Korean speaker, so subtle prosodic nuances may be missed.", "Evaluator is not a native Korean speaker, but can assess general prosody and text fidelity." ] } }, { "id": "f20-es-ES", "scenario": "Multilingual conversation", "scenarioId": "es-ES", "languageId": "es-ES", "languageCode": "es-ES", "locale": "es-ES", "text": "[warm, reassuring] No tengas prisa. He reservado dos sitios para mañana a las 7:45 de la tarde. Podemos sentarnos junto a la ventana, hablar de tu día y disfrutar de un rato tranquilo. [gently teasing] Pero prométeme que esta vez me dejarás la última galleta.", "language": "Spanish (Spain)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-es-ES-A.wav", "model": "s2-pro", "duration": 15.8098 }, "b": { "src": "audio/f20-es-ES-B.wav", "model": "s2.1-pro", "duration": 15.9724 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit a very strong English accent when speaking Spanish, severely impacting naturalness and intelligibility. Neither successfully captures the requested emotional delivery.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit a very strong English accent when speaking Spanish, severely impacting naturalness and intelligibility. Neither successfully captures the requested emotional delivery." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.8, "rationale": "Both samples suffer from a very strong non-native accent that impacts naturalness. Sample A is preferred because Sample B omits a word from the target text." } ], "scores": { "A": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 }, "B": { "naturalness": 2, "intelligibility": 3, "expressiveness": 2.5, "audioQuality": 3.5 } }, "fidelity": { "A": "3.5/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'mañana' pronounced without the 'ñ' sound.", "severity": "major", "timestamp_seconds": 3.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, missing the 'warm, reassuring' and 'gently teasing' cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "text_omission", "evidence": "Omitted the word 'dos' (heard 'He reservado sitios').", "severity": "major", "timestamp_seconds": 1.5 }, { "order": "BA", "category": "pronunciation", "evidence": "Very strong English accent throughout.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Heavy English accent throughout, e.g., 'mañana' pronounced without the 'ñ' sound.", "severity": "major", "timestamp_seconds": 4.0 }, { "order": "AB", "category": "delivery_mismatch", "evidence": "Delivery is flat and robotic, missing the 'warm, reassuring' and 'gently teasing' cues.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "pronunciation", "evidence": "Very strong English accent throughout the recording.", "severity": "major", "timestamp_seconds": null } ] }, "limitations": [ "Both samples feature a very strong non-native accent, making it difficult to assess native-level prosody and expressiveness." ] } }, { "id": "f20-fr-FR", "scenario": "Multilingual conversation", "scenarioId": "fr-FR", "languageId": "fr-FR", "languageCode": "fr-FR", "locale": "fr-FR", "text": "[warm, reassuring] Prends ton temps. J’ai réservé deux places pour demain soir à 19 h 45. On pourra s’installer près de la fenêtre, parler de ta journée et profiter d’un moment tranquille. [gently teasing] Mais promets-moi de me laisser le dernier biscuit cette fois.", "language": "French (France)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-fr-FR-A.wav", "model": "s2.1-pro", "duration": 15.3687 }, "b": { "src": "audio/f20-fr-FR-B.wav", "model": "s2-pro", "duration": 15.0204 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text with a similar non-native accent and minor pronunciation quirks. Neither clearly outperforms the other in naturalness or expressiveness.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text with a similar non-native accent and minor pronunciation quirks. Neither clearly outperforms the other in naturalness or expressiveness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural and fluid delivery with better audio quality. Sample A has noticeable background hiss and slightly robotic prosody." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Slight mispronunciation of 'Prends' with a non-native vowel sound.", "severity": "minor", "timestamp_seconds": 0.5 } ], "B": [ { "order": "AB", "category": "pronunciation", "evidence": "Slight mispronunciation of 'Prends' with a non-native vowel sound.", "severity": "minor", "timestamp_seconds": 0.5 }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly disjointed cadence and unnatural pauses.", "severity": "minor", "timestamp_seconds": 6.0 }, { "order": "BA", "category": "distortion", "evidence": "Noticeable background hiss throughout the recording.", "severity": "minor", "timestamp_seconds": 0.0 } ] }, "limitations": [ "Both samples exhibit a noticeable non-native accent, making it difficult to assess perfect native pronunciation." ] } }, { "id": "f20-de-DE", "scenario": "Multilingual conversation", "scenarioId": "de-DE", "languageId": "de-DE", "languageCode": "de-DE", "locale": "de-DE", "text": "[warm, reassuring] Lass dir Zeit. Ich habe für morgen Abend um 19:45 Uhr zwei Plätze reserviert. Wir können am Fenster sitzen, über deinen Tag reden und gemeinsam einen ruhigen Moment genießen. [gently teasing] Aber versprich mir, dass du mir diesmal den letzten Keks übrig lässt.", "language": "German (Germany)", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-de-DE-A.wav", "model": "s2-pro", "duration": 17.3424 }, "b": { "src": "audio/f20-de-DE-B.wav", "model": "s2.1-pro", "duration": 19.4322 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples deliver the text accurately with similar levels of warmth and expressiveness. Both exhibit minor synthetic artifacts and breathiness, making them comparable in overall quality.", "confidence": 0.75, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples deliver the text accurately with similar levels of warmth and expressiveness. Both exhibit minor synthetic artifacts and breathiness, making them comparable in overall quality." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "A", "canonicalPreference": "B", "confidence": 0.7, "rationale": "Sample A offers slightly better audio quality and a more natural cadence. Sample B sounds somewhat muffled and stiff in its transitions." } ], "scores": { "A": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 4, "audioQuality": 3.5 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural cadence and breathiness throughout the sample.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "distortion", "evidence": "Audio has a slightly muffled, hissy quality.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Stiff delivery and transitions.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Minor synthetic breathiness and slightly unnatural phrasing.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses between sentences.", "severity": "minor", "timestamp_seconds": 7.0 } ] }, "limitations": [ "Subtle nuances in German teasing tone may be subjective." ] } }, { "id": "f20-pt-BR", "scenario": "Multilingual conversation", "scenarioId": "pt-BR", "languageId": "pt-BR", "languageCode": "pt-BR", "locale": "pt-BR", "text": "[warm, reassuring] Não precisa ter pressa. Reservei dois lugares para amanhã, às 7:45 da noite. Podemos sentar perto da janela, conversar sobre o seu dia e aproveitar um momento tranquilo juntos. [gently teasing] Só me promete que desta vez vai deixar o último biscoito pra mim.", "language": "Brazilian Portuguese", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-pt-BR-A.wav", "model": "s2-pro", "duration": 18.8517 }, "b": { "src": "audio/f20-pt-BR-B.wav", "model": "s2.1-pro", "duration": 18.6427 }, "judgment": { "winner": "A", "reason": "Both orders prefer gallery A. First-order review: Sample A offers slightly more natural phrasing and prosody. Sample B sounds more choppy and robotic in its cadence.", "confidence": 0.75, "agreement": "Both presentation orders agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "A", "canonicalPreference": "A", "confidence": 0.7, "rationale": "Sample A offers slightly more natural phrasing and prosody. Sample B sounds more choppy and robotic in its cadence." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Both samples fail to use the requested Brazilian Portuguese accent, sounding distinctly European. Sample B is preferred for its significantly better naturalness, smoother prosody, and clearer audio quality." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 4 }, "B": { "naturalness": 2, "intelligibility": 4, "expressiveness": 3, "audioQuality": 3.5 } }, "fidelity": { "A": "4.0/5 mean text-fidelity score", "B": "4.0/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "pronunciation", "evidence": "Accent leans towards European Portuguese or non-native rather than standard Brazilian Portuguese.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Speaker uses strong European Portuguese pronunciation traits rather than the requested Brazilian Portuguese locale.", "severity": "major", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Phrasing is somewhat choppy and lacks natural conversational flow.", "severity": "major", "timestamp_seconds": null }, { "order": "AB", "category": "pronunciation", "evidence": "Accent exhibits non-native or European Portuguese traits.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "wrong_language", "evidence": "Speaker uses European Portuguese pronunciation (e.g., palatalized 's' in 'lugares', unpalatalized 'd' in 'dia') instead of pt-BR.", "severity": "major", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "The pacing is somewhat disjointed and lacks natural conversational flow.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Both samples exhibit non-native or European Portuguese accent features despite the pt-BR locale request.", "Both samples exhibit European Portuguese pronunciation traits rather than the requested Brazilian Portuguese locale." ] } }, { "id": "f20-ar", "scenario": "Multilingual conversation", "scenarioId": "ar", "languageId": "ar", "languageCode": "ar", "locale": "ar", "text": "[warm, reassuring] لا داعي للاستعجال. حجزت مقعدين للغد في الساعة 7:45 مساءً. يمكننا الجلوس قرب النافذة، والحديث عن يومك، والاستمتاع بلحظة هادئة معًا. [gently teasing] لكن لديّ طلب صغير: لنترك آخر قطعة بسكويت لي هذه المرة، اتفقنا؟", "language": "Modern Standard Arabic", "direction": "Begin warm and reassuring, then shift to gentle teasing at the second cue. Read the entire passage in the specified language. Bracket cues are performance directions, not spoken words.", "englishGloss": "Take your time. I've reserved two seats for tomorrow at 7:45 in the evening. We can sit by the window, talk about your day, and enjoy a quiet moment together. Just promise you'll leave me the last cookie this time.", "a": { "src": "audio/f20-ar-A.wav", "model": "s2-pro", "duration": 21.8935 }, "b": { "src": "audio/f20-ar-B.wav", "model": "s2.1-pro", "duration": 21.3594 }, "judgment": { "winner": "inconsistent", "reason": "No stable preference across the two presentation orders. First-order review: Both samples exhibit similar synthetic qualities and minor unnatural pauses. Neither sample stands out as significantly better in terms of naturalness or expressiveness.", "confidence": 0.8, "agreement": "Presentation orders do not agree", "reviews": 2, "orderReviews": [ { "order": "AB", "mapping": "Judge A = gallery A; judge B = gallery B", "preference": "tie", "canonicalPreference": "tie", "confidence": 0.8, "rationale": "Both samples exhibit similar synthetic qualities and minor unnatural pauses. Neither sample stands out as significantly better in terms of naturalness or expressiveness." }, { "order": "BA", "mapping": "Judge A = gallery B; judge B = gallery A", "preference": "B", "canonicalPreference": "A", "confidence": 0.8, "rationale": "Sample B provides a more natural and fluid delivery with better audio quality. Sample A suffers from noticeable breathy artifacts and slightly unnatural pauses." } ], "scores": { "A": { "naturalness": 3.5, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3.5 }, "B": { "naturalness": 3, "intelligibility": 4, "expressiveness": 3.5, "audioQuality": 3 } }, "fidelity": { "A": "4.5/5 mean text-fidelity score", "B": "4.5/5 mean text-fidelity score" }, "issues": { "A": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Slightly unnatural pacing and pauses throughout the utterance.", "severity": "minor", "timestamp_seconds": null } ], "B": [ { "order": "AB", "category": "robotic_prosody", "evidence": "Unnatural pacing and a slightly synthetic tone.", "severity": "minor", "timestamp_seconds": null }, { "order": "BA", "category": "robotic_prosody", "evidence": "Slightly unnatural pauses and breathy artifacts throughout the recording.", "severity": "minor", "timestamp_seconds": null } ] }, "limitations": [ "Evaluated based on general Modern Standard Arabic pronunciation.", "I am not a native speaker of Modern Standard Arabic, so subtle pronunciation nuances may not be fully captured." ] } } ] } ] }