From 30c9cf4ba3154c551e9f4a2172f8f4486db89023 Mon Sep 17 00:00:00 2001
From: slonkazoid <slonkazoid@slonk.ing>
Date: Sat, 18 Jan 2025 22:49:41 +0300
Subject: [PATCH] add simple deduper script

---
 bash/dedupe.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100755 bash/dedupe.sh

diff --git a/bash/dedupe.sh b/bash/dedupe.sh
new file mode 100755
index 0000000..49a420e
--- /dev/null
+++ b/bash/dedupe.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+shopt -s globstar
+
+if [[ -n "$1" ]]; then cd "$1"; fi
+
+declare -A occurences
+
+for file in **/*; do
+    if [[ ! -f "$file" || -L "$file" ]]; then continue; fi
+    hash=$(xxh128sum "$file" | cut -f1 -d' ')
+    occurence=${occurences[$hash]}
+    if [[ -n "$occurence" ]]; then
+        relpath=$(realpath -s --relative-to "$(dirname "$file")" "$occurence")
+        ln -sf "$relpath" "$file"
+        echo "symlinked ${file@Q} to ${relpath@Q}" >&2
+    else
+        echo "first occurence of $hash: ${file@Q}" >&2
+        occurences[$hash]=$file
+    fi
+done