Unique links

Cargo.lock

@@ -677,6 +677,21 @@ version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linked_hash_set"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
@@ -1438,6 +1453,7 @@ dependencies = [
  "anyhow",
  "clap",
  "env_logger",
+ "linked_hash_set",
  "log",
  "regex",
  "reqwest",

Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 anyhow = "1.0.86"
 clap = { version = "4.5.13", features = ["derive"] }
 env_logger = "0.11.5"
+linked_hash_set = "0.1.4"
 log = "0.4.22"
 regex = "1.10.6"
 reqwest = { version = "0.12.5", features = ["json"] }

src/main.rs

@@ -1,6 +1,7 @@
 use std::{env, fmt::Display};
 
 use clap;
+use linked_hash_set::{self, LinkedHashSet};
 use log::{error, info, LevelFilter};
 use svtl::{get_base_url, get_content, get_relative_links, YtDlpWrapper};
 
@@ -53,8 +54,11 @@ async fn main() {
         }
     };
 
-    let links = get_relative_links(&content);
+    let aux_for_unique =
-    info!("Found the following links: \n{}", join_long(&links));
+        LinkedHashSet::<String>::from_iter(get_relative_links(&content).into_iter());
+    let links = aux_for_unique.into_iter().collect::<Vec<_>>();
+
+    info!("Found the following unqiue links: \n{}", join_long(&links));
 
     for link_part in links {
         let link = format!("{}{}", base_url, link_part);