{ "schemaVersion": "1.1", "id": "deepswapai-face-swap-evaluation-metric-map-v1", "identifier": "deepswapai-face-swap-evaluation-metric-map-v1", "version": "1.1.0", "title": "DeepSwapAI face swap evaluation metric decision map", "alternateName": "Face swap benchmark metric selection matrix", "description": "A source-cited decision map and benchmark-protocol crosswalk for selecting face swap identity, target-attribute, distribution, temporal, no-reference, or human-review evidence without combining incompatible protocols.", "canonicalUrl": "https://deepswapai.com/face-swap-quality-scorecard#automated-benchmark-map", "datePublished": "2026-07-22", "dateModified": "2026-07-28", "creator": "DeepSwapAI Product Team", "isAccessibleForFree": true, "license": "https://creativecommons.org/licenses/by/4.0/", "attribution": "DeepSwapAI Face Swap Evaluation Metric Decision Map v1, https://deepswapai.com/face-swap-quality-scorecard#automated-benchmark-map", "dataCatalogUrl": "https://deepswapai.com/research#catalog", "citationFileUrl": "https://deepswapai.com/assets/evaluation-metrics/face-swap-evaluation-metric-map-v1.bib", "scope": "A literature-grounded selection map describing what nine evaluation families compare, the protocol level they require, and how three published face swap benchmark protocols differ.", "evidenceBoundary": [ "This map does not run any metric, score any face swap output, or compare any provider.", "A metric name does not make values portable across datasets, evaluators, crops, preprocessing, implementations, sample counts, or aggregation rules.", "Identity, target-attribute preservation, distribution realism, temporal behavior, no-reference quality, and human review answer different questions.", "Protocol rows summarize paper-reported setups; they do not reproduce benchmark results, establish dataset availability, or make scores comparable across papers.", "Human review, consent, disclosure, and use-specific acceptance remain separate from an automated benchmark value." ], "protocolChecklist": [ "Name the source, target, output, reference set, or frame sequence compared by each measure.", "Pin the dataset version, evaluator implementation and weights, crop, alignment, color pipeline, and preprocessing.", "Report sample count, repeated runs, failed samples, aggregation, dispersion, and confidence intervals where applicable.", "Keep identity transfer, target preservation, set-level realism, temporal behavior, and human acceptance as separately inspectable results.", "Treat benchmark challenge slices as protocol definitions, not as interchangeable score columns across datasets." ], "benchmarkProtocols": [ { "id": "casia-faceswapping", "name": "CASIA FaceSwapping", "primarySource": "https://arxiv.org/abs/2605.00883", "mediaType": "video", "comparisonUnit": "Source-target video pairs assembled under three factor-isolation protocols.", "reportedScale": [ "Normal: 4,500 non-overlapping same-ethnicity pairs from normal recordings.", "Cross-ethnicity: 1,200 pairs spanning both directions between Asian, African, and Caucasian groups.", "Cross-attribute: 4,300 pairs spanning normal and pose, expression, or illumination variations in both directions." ], "challengeSlices": [ "normal conditions", "cross-ethnicity generalization", "pose variation", "expression variation", "illumination variation" ], "reportedMeasures": [ "identity retrieval", "identity similarity", "pose error", "expression error", "FID", "subject consistency", "background consistency" ], "humanReview": "No human-review protocol is summarized in this crosswalk.", "boundary": "The three slices isolate different factors inside the CASIA setup. Their values depend on that paper's pair construction, models, feature extractors, resolutions, and aggregation." }, { "id": "idbench-v", "name": "IDBench-V", "primarySource": "https://arxiv.org/abs/2601.01425", "mediaType": "video", "comparisonUnit": "200 paper-reported real-world source-video and target-image pairs.", "reportedScale": [ "One 200-pair evaluation set covering small faces, extreme head poses, severe occlusions, complex or dynamic expressions, and cluttered multi-person scenes." ], "challengeSlices": [ "small faces", "extreme head pose", "severe occlusion", "dynamic expression", "multi-person scenes" ], "reportedMeasures": [ "ArcFace identity similarity", "InsightFace identity similarity", "CurricularFace identity similarity", "frame-wise identity-similarity variance", "pose error", "expression error", "background consistency", "subject consistency", "motion smoothness", "FVD" ], "humanReview": "The paper reports 19 evaluators using 1-to-5 ratings for identity similarity, attribute preservation, and video quality.", "boundary": "The row records the paper-reported protocol only. It does not assert public download availability, reproduce a result, or make its values portable to another benchmark." }, { "id": "canonswap-vfs", "name": "CanonSwap VFS benchmark", "primarySource": "https://arxiv.org/abs/2507.02691", "mediaType": "video-with-audio", "comparisonUnit": "100 source-target pairs sampled from VFHQ; each target uses the first 100 frames and four seconds of corresponding audio.", "reportedScale": [ "One 100-pair video face swap evaluation set with fixed frame and audio windows." ], "challengeSlices": [ "global identity and target attributes", "eye dynamics", "audio-lip synchronization", "optical-flow temporal consistency", "video distribution quality" ], "reportedMeasures": [ "identity similarity", "identity retrieval", "pose error", "expression error", "gaze error", "eye aspect ratio error", "LSE-D", "LSE-C", "optical-flow temporal consistency", "FVD" ], "humanReview": "No human-review protocol is summarized in this crosswalk.", "boundary": "The protocol uses paper-specific frame, audio, estimator, and feature-extractor choices. Its values must not be compared directly with IDBench-V or CASIA values." } ], "metrics": [ { "id": "identity-retrieval-similarity", "name": "Identity retrieval or embedding similarity", "category": "source-identity-transfer", "mediaTypes": [ "photo", "video", "gif" ], "protocolLevel": "dataset", "compares": "Source identity versus the swapped output, using a named face encoder or source gallery.", "question": "How strongly does the output retain source-identity evidence under the fixed evaluator?", "direction": "Higher similarity or retrieval performance is usually better inside the same protocol.", "prerequisites": [ "Fixed source gallery", "Named encoder and weights", "Documented detection, alignment, crop, and aggregation" ], "boundary": "The encoder, crop, gallery, demographics, threshold, and preprocessing affect the value. It is not a biometric verdict for one person.", "primarySources": [ "https://arxiv.org/abs/2605.00883", "https://arxiv.org/abs/2601.01425" ] }, { "id": "framewise-identity-stability", "name": "Frame-wise identity similarity stability", "category": "temporal-identity-consistency", "mediaTypes": [ "video", "gif" ], "protocolLevel": "sequence-dataset", "compares": "The sequence of source-to-output identity similarities across detected output frames under one named face encoder.", "question": "Does source-identity evidence remain stable as pose, expression, occlusion, and scene conditions change over time?", "direction": "Lower dispersion can indicate greater stability only when the mean identity similarity or retrieval result is reported beside it.", "prerequisites": [ "Fixed source identity and sequence set", "Named face encoder, weights, detection, alignment, and crop", "Documented missing-frame policy, frame sampling, dispersion statistic, and aggregation" ], "boundary": "A consistently wrong identity can have low variance. Dispersion cannot replace mean similarity or retrieval, and values are not portable across encoders or frame pipelines.", "primarySources": [ "https://arxiv.org/abs/2601.01425" ] }, { "id": "pose-expression-error", "name": "Pose and expression error", "category": "target-attribute-preservation", "mediaTypes": [ "photo", "video", "gif" ], "protocolLevel": "dataset", "compares": "Target pose or expression estimates versus the swapped output under a named estimator.", "question": "How closely does the output preserve target head pose and expression under the fixed estimator?", "direction": "Lower error is usually better inside the same estimator and parameterization.", "prerequisites": [ "Fixed target-output pairs", "Named pose or expression estimator", "Documented parameter space, crop, and aggregation" ], "boundary": "Values are not portable across different estimators, crops, parameter spaces, landmark conventions, or preprocessing pipelines.", "primarySources": [ "https://arxiv.org/abs/2605.00883", "https://arxiv.org/abs/2601.01425" ] }, { "id": "frechet-inception-distance", "name": "Frechet Inception Distance (FID)", "category": "set-level-distribution", "mediaTypes": [ "photo", "video-frame-set", "gif-frame-set" ], "protocolLevel": "dataset", "compares": "A distribution of generated images or frames versus a documented reference distribution.", "question": "How close are two feature distributions under the fixed feature extractor and sampling protocol?", "direction": "Lower is better only inside the same dataset, extractor, preprocessing, and sample protocol.", "prerequisites": [ "Generated and reference image sets", "Fixed feature extractor", "Documented sample count, resize, color, and aggregation" ], "boundary": "One face swap output cannot have a defensible FID because a single image is not a distribution. FID does not isolate identity correctness.", "primarySources": [ "https://arxiv.org/abs/1706.08500", "https://arxiv.org/abs/2605.00883" ] }, { "id": "frechet-video-distance", "name": "Frechet Video Distance (FVD)", "category": "set-level-video-distribution", "mediaTypes": [ "video" ], "protocolLevel": "sequence-dataset", "compares": "A distribution of generated video clips versus a documented reference-clip distribution under a named video feature extractor.", "question": "How close are generated and reference video feature distributions under one fixed clip-sampling protocol?", "direction": "Lower is better only inside the same dataset, feature extractor, clip duration, frame rate, preprocessing, and sample protocol.", "prerequisites": [ "Generated and reference clip sets", "Named video feature extractor and implementation", "Documented clip count, duration, frame rate, frame sampling, resize, preprocessing, and aggregation" ], "boundary": "One output clip is not a defensible distribution-level FVD result. FVD does not isolate source identity, target-attribute preservation, lip synchronization, or a specific temporal defect.", "primarySources": [ "https://arxiv.org/abs/2601.01425", "https://arxiv.org/abs/2507.02691" ] }, { "id": "subject-background-consistency", "name": "Subject and background consistency", "category": "temporal-consistency", "mediaTypes": [ "video", "gif" ], "protocolLevel": "sequence-dataset", "compares": "Feature consistency across output frames and against preceding, target, or reference frames.", "question": "How stable are subject and background features through motion under the fixed video protocol?", "direction": "Higher consistency is usually better under the same implementation.", "prerequisites": [ "Fixed frame sampling", "Named feature extractor", "Documented reference relationship and temporal aggregation" ], "boundary": "Consistency alone cannot establish correct identity mapping, visual realism, motion accuracy, or publication readiness.", "primarySources": [ "https://arxiv.org/abs/2605.00883", "https://arxiv.org/abs/2601.01425" ] }, { "id": "fine-grained-video-synchronization", "name": "Gaze, eye aperture, lip sync, and motion diagnostics", "category": "fine-grained-video-behavior", "mediaTypes": [ "video" ], "protocolLevel": "sequence-dataset", "compares": "Named eye, audio-lip, landmark, or optical-flow behavior between target and output sequences.", "question": "Which specific dynamic target attributes remain synchronized through the generated video?", "direction": "Direction and unit depend on the named diagnostic; report each separately.", "prerequisites": [ "Fixed frame and audio alignment", "Named estimator and implementation", "Documented units, missing-frame policy, and aggregation" ], "boundary": "These diagnostics are not interchangeable, need a documented video protocol, and do not apply to every media type.", "primarySources": [ "https://arxiv.org/abs/2507.02691" ] }, { "id": "learned-no-reference-assessment", "name": "Learned no-reference quality assessment", "category": "output-only-model-assessment", "mediaTypes": [ "photo" ], "protocolLevel": "trained-model-plus-test-set", "compares": "The output alone, evaluated by a model trained against ranked human judgments or labeled quality data.", "question": "How does a trained assessor rank visible output quality when explicit reference media are unavailable?", "direction": "Direction depends on the model output; validate it on a documented held-out distribution.", "prerequisites": [ "Named model and weights", "Documented training and test distributions", "Fixed preprocessing and validation protocol" ], "boundary": "Performance depends on training data and validation scope. An output-only score cannot establish correct source identity transfer by itself.", "primarySources": [ "https://arxiv.org/abs/2406.01884" ] }, { "id": "structured-human-scorecard", "name": "Structured human scorecard", "category": "human-review", "mediaTypes": [ "photo", "video", "gif" ], "protocolLevel": "single-output-or-study", "compares": "One visible output against explicit defect anchors, publication gates, and reviewer notes.", "question": "Which visible defects or unresolved use gates does a reviewer observe in this specific output?", "direction": "Use anchored ordinal ratings; do not reinterpret the result as a probability or biometric score.", "prerequisites": [ "Documented viewing conditions", "Explicit scale anchors", "Applicable criteria, reviewer notes, and unresolved gates" ], "boundary": "Subjective evidence about the reviewed sample, not model accuracy, provider ranking, probability, or biometric similarity.", "primarySources": [ "https://www.itu.int/rec/R-REC-BT.500-15-202305-I", "https://www.itu.int/rec/T-REC-P.910/en" ] } ], "references": [ { "name": "Towards High Fidelity Face Swapping: A Comprehensive Survey and New Benchmark", "url": "https://arxiv.org/abs/2605.00883", "relationship": "Defines the CASIA FaceSwapping benchmark and documents identity, target-attribute, distribution, and video consistency protocols." }, { "name": "DreamID-V: Bridging the Image-to-Video Gap for High-Fidelity Face Swapping via Diffusion Transformer", "url": "https://arxiv.org/abs/2601.01425", "relationship": "Introduces the 200-pair IDBench-V protocol and reports multi-encoder identity similarity, frame-wise variance, target-attribute, video-consistency, FVD, and human-review measures." }, { "name": "CanonSwap: High-Fidelity and Consistent Video Face Swapping via Canonical Space Modulation", "url": "https://arxiv.org/abs/2507.02691", "relationship": "Reports video-specific identity, target-attribute, synchronization, temporal consistency, and distribution measures." }, { "name": "Rank-based No-reference Quality Assessment for Face Swapping", "url": "https://arxiv.org/abs/2406.01884", "relationship": "Studies learned output-only face-swap quality assessment trained against ranked judgments." }, { "name": "GANs Trained by a Two Time-Scale Update Rule Converge to a Local Nash Equilibrium", "url": "https://arxiv.org/abs/1706.08500", "relationship": "Introduces Frechet Inception Distance as a distribution-level evaluation measure." }, { "name": "ITU-R BT.500-15", "url": "https://www.itu.int/rec/R-REC-BT.500-15-202305-I", "relationship": "Primary standard for documented subjective image assessment methodology." }, { "name": "ITU-T P.910 (10/2023)", "url": "https://www.itu.int/rec/T-REC-P.910/en", "relationship": "Primary standard for documented subjective multimedia assessment methodology." } ] }