|
<!DOCTYPE html> |
|
<html lang="en"> |
|
<head> |
|
<meta charset="UTF-8" /> |
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
|
<title>Repo-Level Dedupe Visualization</title> |
|
<link rel="stylesheet" href="style.css" /> |
|
<script src="https://cdn.jsdelivr.net/npm/vega@5"></script> |
|
<script src="https://cdn.jsdelivr.net/npm/vega-lite@5"></script> |
|
<script src="https://cdn.jsdelivr.net/npm/vega-embed@6"></script> |
|
</head> |
|
<body> |
|
<div class="container"> |
|
<div class="header"> |
|
<h1>Visualizing Repo-Level Dedupe</h1> |
|
<p> |
|
This visualization demonstrates block-level deduplication across all |
|
models in |
|
<a |
|
target="_blank" |
|
href="https://huggingface.co/bartowski/gemma-2-9b-it-GGUF" |
|
>bartowski/gemma-2-9b-it-GGUF</a |
|
>. |
|
</p> |
|
<p> |
|
Each row represents a file in the repository grouped into blocks of up |
|
to 64MB. The color of each block represents the deduplication ratio |
|
for the block, which is a function of how often the chunks in the |
|
block are shared between files. The darker the color, the more |
|
frequently content is shared, the better the overall upload and |
|
download times for a given file! The deduplication savings here take a |
|
191GB repo and cut it down to 97GB, helping to shave a few hours off |
|
the upload time. |
|
</p> |
|
<p> |
|
You can read more about chunks, blocks, and the nitty gritty details |
|
of how we make this all work in our accompanying |
|
<a |
|
target="_blank" |
|
href="https://huggingface.co/blog/from-chunks-to-blocks" |
|
>blog post</a |
|
>. |
|
</p> |
|
To explore the visualization: |
|
<ul> |
|
<li> |
|
<strong>Hover</strong> over a block in an individual file to |
|
highlight it and see where else it appears in the repository. |
|
</li> |
|
<li> |
|
<strong>Click</strong> any block in a file to see all other files |
|
that share blocks. |
|
</li> |
|
<li> |
|
<strong>Double-click</strong> anywhere on any file to reset and |
|
continue exploring. |
|
</li> |
|
</ul> |
|
</div> |
|
|
|
<div class="heatmap-container"> |
|
<div id="vis"></div> |
|
</div> |
|
</div> |
|
<script> |
|
var vlSpec = { |
|
$schema: "https://vega.github.io/schema/vega-lite/v5.json", |
|
resolve: { scale: { x: "independent" } }, |
|
width: 800, |
|
height: 25, |
|
params: [ |
|
{ |
|
name: "highlight", |
|
select: { type: "point", fields: ["xorb_id"], on: "pointerover" }, |
|
}, |
|
{ |
|
name: "select", |
|
select: { type: "point", fields: ["repo"], toggle: "false" }, |
|
}, |
|
{ |
|
name: "xorbs_selected", |
|
expr: "pluck(data('source_0'), 'repo_xorb_selected')", |
|
}, |
|
{ |
|
name: "any_xorbs_selected", |
|
expr: "extent(xorbs_selected)[0] != null", |
|
}, |
|
], |
|
transform: [ |
|
{ |
|
calculate: |
|
"(select.repo != null ? indexof(select.repo, datum.repo) : -1) + 1", |
|
as: "repo_selected", |
|
}, |
|
{ |
|
calculate: "if(datum.repo_selected > 0, datum.xorb_id, null)", |
|
as: "repo_xorb_selected", |
|
}, |
|
{ |
|
calculate: |
|
"split(datum.repo, '/')[length(split(datum.repo, '/')) - 1]", |
|
as: "repo", |
|
}, |
|
], |
|
data: { |
|
url: "xorbs.json", |
|
}, |
|
mark: "rect", |
|
encoding: { |
|
x: { |
|
field: "xorb_id", |
|
axis: null, |
|
sort: { field: "dedupe_factor", order: "descending" }, |
|
stack: "normalize", |
|
}, |
|
color: { |
|
condition: [ |
|
{ test: "datum.xorb_id == highlight.xorb_id", value: "orange" }, |
|
], |
|
field: "dedupe_factor", |
|
type: "quantitative", |
|
scale: { scheme: "blues", domain: [0, 10] }, |
|
}, |
|
opacity: { |
|
condition: [ |
|
{ |
|
test: "any_xorbs_selected && indexof(xorbs_selected, datum.xorb_id) == -1", |
|
value: 0.2, |
|
}, |
|
], |
|
}, |
|
tooltip: [ |
|
{ field: "repo", type: "nominal", title: "File" }, |
|
{ field: "xorb_id", type: "nominal", title: "Block Hash" }, |
|
{ |
|
field: "dedupe_factor", |
|
type: "quantitative", |
|
title: "Dedupe Factor", |
|
}, |
|
], |
|
row: { |
|
field: "repo", |
|
title: "", |
|
spacing: 1, |
|
header: { labelAngle: 0, labelAlign: "left", labelFontSize: 14 }, |
|
sort: { field: "repo", order: "ascending" }, |
|
}, |
|
}, |
|
}; |
|
vegaEmbed("#vis", vlSpec); |
|
</script> |
|
</body> |
|
</html> |
|
|
