如何重用/将ureq代理传递给我的线程

Question

请考虑一下这个生锈代码，

use scoped_threadpool::Pool;
use std::thread;

const BASE_URL: &str = "https://www.bing.com/search?q=";

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet(&self) -> bool {
        let resp = ureq::get(self.url).timeout_connect(5_000).call();
        if !resp.ok() {
            return false;
        } else {
            return resp.ok();
        }
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(3);

    pool.scoped(|scoped| {
        for entry in our_vec {
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, entry);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };
                let our_bool = this_item.doGet();
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

它生成10个urls，并对每个urls执行一个最小的GET，并且它可以工作。但是，上面的代码为每个url使用了新的连接，因此对我的实际情况来说太昂贵了。ureq似乎有支持连接池的代理。

所以我试着：

const BASE_URL: &str = "https://www.bing.com/search?q=";
use scoped_threadpool::Pool;
use std::thread;

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet(&self) -> bool {
        let resp = ureq::get(self.url).timeout_connect(5_000).call();
        if !resp.ok() {
            return false;
        } else {
            return resp.ok();
        }
    }
    fn doGet_withAgent(&self, some_Agent: &ureq::Agent) -> bool {
        return true;
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(3);
    let agent = ureq::Agent::new();

    pool.scoped(|scoped| {
        for entry in our_vec {
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, 3);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };
                //let our_bool = this_item.doGet();
                let our_bool = this_item.doGet_withAgent(&agent);
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

但似乎无法做到正确：

error[E0382]: use of moved value: `agent`
  --> src/main.rs:33:28
   |
33 |             scoped.execute(move || {
   |                            ^^^^^^^ value moved into closure here, in previous iteration of loop
...
40 |                 let our_bool = this_item.doGet_withAgent(&agent);
   |                                                           ----- use occurs due to use in closure

我的目标是重用我的联系。实现这一目标的惯用方法是什么？

编辑1

const BASE_URL: &str = "https://www.bing.com/search?q=";
use scoped_threadpool::Pool;
use std::thread;

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet(&self) -> bool {
        let resp = ureq::get(self.url).timeout_connect(5_000).call();
        if !resp.ok() {
            return false;
        } else {
            return resp.ok();
        }
    }
    fn doGet_withAgent(&self, some_Agent: &ureq::Agent) -> bool {
        let resp = some_Agent.get(self.url).timeout_connect(5_000).call();
        return true;
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(3);
    let mut agent = ureq::Agent::new();

    pool.scoped(|scoped| {
        for entry in our_vec {
            let agent = &agent;
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, entry);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };
                let our_bool = this_item.doGet_withAgent(agent);
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

编辑2

因此，我将代理封装在一个ARC中，在连续的请求中仍然使用新的连接。

const BASE_URL: &str = "https://www.google.com/search?q=";
use scoped_threadpool::Pool;
use std::sync::Arc;
use std::sync::Mutex;
use std::thread;

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet(&self) -> bool {
        let resp = ureq::get(self.url).timeout_connect(5_000).call();
        return resp.ok();
    }

    //fn doGet_withAgent(&self, some_Agent: ureq::Agent) -> bool {
    //fn doGet_withAgent(&self, some_mutex: &std::sync::Arc)-> bool{
    fn doGet_withAgent(&self, some_mutex: Arc<Mutex<ureq::Agent>>) -> bool {
        let mut guard = some_mutex.lock().unwrap();
        let resp = guard.head(self.url).timeout_connect(5_000).call();
        return resp.ok();
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(3);
    let agent = Arc::new(Mutex::new(ureq::Agent::new()));

    pool.scoped(|scoped| {
        for entry in our_vec {
            let agent_clone = agent.clone();
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, 3);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };
                let our_bool = this_item.doGet_withAgent(agent_clone);
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

Answer 1

最后，我从ureq转到了ureq

Cargo.toml

[dependencies]
scoped_threadpool = "0.1.9"
reqwest = { version = "0.10", features = ["blocking"] }
...

const BASE_URL: &str = "https://www.BinG.com/search?q=";
use scoped_threadpool::Pool;
use std::sync::Arc;
use std::sync::Mutex;
use std::thread;

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet_withAgent(&self, some_mutex: Arc<Mutex<reqwest::blocking::Client>>) -> bool {
        let resp;
        let mut guard = some_mutex.lock().unwrap();
        match guard.head(self.url).send() {
            Ok(in_here) => {
                resp = in_here;
            }
            _ => {
                return false;
            }
        }

        match resp.status().as_u16() {
            200 => {
                return true;
            }
            _ => (),
        }

        return false;
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(2);

    let agent = Arc::new(Mutex::new(reqwest::blocking::Client::new()));
    pool.scoped(|scoped| {
        for entry in our_vec {
            let agent_clone = agent.clone();
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, entry);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };

                let our_bool = this_item.doGet_withAgent(agent_clone);
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

它在连接重用方面表现良好，尽管我不确定线程对此有多大帮助：

Hello, world!
1 status: true
2 status: true
3 status: true
4 status: true
5 status: true
6 status: true
7 status: true
8 status: true
9 status: true
10 status: true

执行似乎是连续的，不确定如果切换到async而不是blocking reqwest，是否会看到不同之处。

Answer 2

与戈朗不同，在那里，http客户端是安全，供多个用户同时使用--我已经了解到，每个锈蚀实现都不可能是线程支持的/安全的。

然后找到了isahc，它的HttpClient完全是线程安全的。简单得多，不需要ARC互斥对象，因为它可以对client进行静默引用：

Cargo.toml

[dependencies]
scoped_threadpool = "0.1.9"
isahc = "0.8.2"

代码：

const BASE_URL: &str = "https://www.BinG.com/search?q=";
use scoped_threadpool::Pool;
use isahc::prelude::*;

struct my_struct<'a> {
    url: &'a str,
    id: i16,
}

impl my_struct<'_> {
    fn doGet_withClient(&self, incoming_client: &isahc::HttpClient) -> bool {
        let response;
        match incoming_client.get(self.url) {
            Ok(resp) => {
                response = resp;
            }
            _ => {
                return false;
            }
        }
        return response.status().is_success();

        return false;
    }
}

fn main() {
    println!("Hello, world!");
    let our_vec = (1..11).collect::<Vec<_>>();
    let mut pool = Pool::new(4);
    let client;
    match HttpClient::new() {
        Ok(this_thing) => {
            client = this_thing;
        }
        _ => {
            panic!("could not get http client thingy");
        }
    }

    pool.scoped(|scoped| {
        for entry in our_vec {
            let new_client = &client;
            scoped.execute(move || {
                let url = format!("{}{}", BASE_URL, entry);
                let this_item = my_struct {
                    url: &url,
                    id: entry,
                };

                let our_bool = this_item.doGet_withClient(new_client);
                println!("{} status: {}", entry, our_bool);
            });
        }
    });
}

运行它：

Hello, world!
3 status: true
4 status: true
2 status: true
1 status: true
5 status: true
6 status: true
8 status: true
9 status: true
10 status: true
7 status: true