From 0dc6f09a7421307e51a0ca9cf5453d5d8f80b3bd Mon Sep 17 00:00:00 2001
From: Love Billenius <lovebillenius@disroot.org>
Date: Tue, 6 Aug 2024 16:49:15 +0200
Subject: [PATCH] Unique links

---
 Cargo.lock  | 16 ++++++++++++++++
 Cargo.toml  |  1 +
 src/main.rs |  8 ++++++--
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7775498..cf6d4f5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -677,6 +677,21 @@ version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linked_hash_set"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
@@ -1438,6 +1453,7 @@ dependencies = [
  "anyhow",
  "clap",
  "env_logger",
+ "linked_hash_set",
  "log",
  "regex",
  "reqwest",
diff --git a/Cargo.toml b/Cargo.toml
index e64f07a..9ae3deb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 anyhow = "1.0.86"
 clap = { version = "4.5.13", features = ["derive"] }
 env_logger = "0.11.5"
+linked_hash_set = "0.1.4"
 log = "0.4.22"
 regex = "1.10.6"
 reqwest = { version = "0.12.5", features = ["json"] }
diff --git a/src/main.rs b/src/main.rs
index acab939..e090de5 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,6 +1,7 @@
 use std::{env, fmt::Display};
 
 use clap;
+use linked_hash_set::{self, LinkedHashSet};
 use log::{error, info, LevelFilter};
 use svtl::{get_base_url, get_content, get_relative_links, YtDlpWrapper};
 
@@ -53,8 +54,11 @@ async fn main() {
         }
     };
 
-    let links = get_relative_links(&content);
-    info!("Found the following links: \n{}", join_long(&links));
+    let aux_for_unique =
+        LinkedHashSet::<String>::from_iter(get_relative_links(&content).into_iter());
+    let links = aux_for_unique.into_iter().collect::<Vec<_>>();
+
+    info!("Found the following unqiue links: \n{}", join_long(&links));
 
     for link_part in links {
         let link = format!("{}{}", base_url, link_part);