Unique links

Cargo.lock

@@ -677,6 +677,21 @@ version = "0.2.155"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linked_hash_set"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
@@ -1438,6 +1453,7 @@ dependencies = [
  "anyhow",
  "clap",
  "env_logger",
+ "linked_hash_set",
  "log",
  "regex",
  "reqwest",

Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 anyhow = "1.0.86"
 clap = { version = "4.5.13", features = ["derive"] }
 env_logger = "0.11.5"
+linked_hash_set = "0.1.4"
 log = "0.4.22"
 regex = "1.10.6"
 reqwest = { version = "0.12.5", features = ["json"] }

src/main.rs

@@ -1,10 +1,21 @@
+use std::{env, fmt::Display};
+
 use clap;
-use log::{error, info};
+use linked_hash_set::{self, LinkedHashSet};
+use log::{error, info, LevelFilter};
 use svtl::{get_base_url, get_content, get_relative_links, YtDlpWrapper};
 
+fn init_logger() {
+    let mut builder = env_logger::Builder::from_env("RUST_LOG");
+    if env::var("RUST_LOG").is_err() {
+        builder.filter_level(LevelFilter::Info);
+    }
+    builder.init();
+}
+
 #[tokio::main]
 async fn main() {
-    env_logger::init();
+    init_logger();
 
     let matches = clap::Command::new("Svt Links")
         .version("1.0")
@@ -43,8 +54,11 @@ async fn main() {
         }
     };
 
-    let links = get_relative_links(&content);
+    let aux_for_unique =
-    info!("Found the following links: {}", links.join(", "));
+        LinkedHashSet::<String>::from_iter(get_relative_links(&content).into_iter());
+    let links = aux_for_unique.into_iter().collect::<Vec<_>>();
+
+    info!("Found the following unqiue links: \n{}", join_long(&links));
 
     for link_part in links {
         let link = format!("{}{}", base_url, link_part);
@@ -55,3 +69,16 @@ async fn main() {
         }
     }
 }
+
+fn join_long<S: Display>(strings: &Vec<S>) -> String {
+    let mut ret = String::new();
+    for (idx, s) in strings.iter().enumerate() {
+        let mut to_push = format!(" {}: {}", idx + 1, s);
+        if idx != strings.len() - 1 {
+            to_push.push('\n');
+        }
+        ret.push_str(&to_push);
+    }
+
+    ret
+}