Upgrade dependencies and enable tokio macros
This commit is contained in:
parent
301cb4da31
commit
f6298eef21
809
Cargo.lock
generated
809
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
10
Cargo.toml
10
Cargo.toml
@ -1,12 +1,12 @@
|
||||
[package]
|
||||
name = "quote-scrape"
|
||||
name = "quote-scraper"
|
||||
version = "0.1.0"
|
||||
authors = ["hr567 <hr567@hr567.me>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
lazy_static = "1.4.0"
|
||||
reqwest = "0.11.0"
|
||||
scraper = "0.12.0"
|
||||
tokio = { version = "1.0.2", features = [ "rt-multi-thread", "sync" ] }
|
||||
url = "2.2.0"
|
||||
reqwest = "0.11.11"
|
||||
scraper = "0.13.0"
|
||||
tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread", "sync"] }
|
||||
url = "2.2.2"
|
||||
|
14
src/main.rs
14
src/main.rs
@ -3,10 +3,7 @@ use std::sync::Arc;
|
||||
use lazy_static::lazy_static;
|
||||
use reqwest::Client;
|
||||
use scraper::{Html, Selector};
|
||||
use tokio::{
|
||||
runtime::Runtime,
|
||||
sync::{mpsc, Semaphore},
|
||||
};
|
||||
use tokio::sync::{mpsc, Semaphore};
|
||||
use url::Url;
|
||||
|
||||
const MAX_TASK: usize = 16;
|
||||
@ -24,6 +21,7 @@ lazy_static! {
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
struct Quote {
|
||||
text: String,
|
||||
@ -54,15 +52,15 @@ fn parse_quote_html(page: Html) -> Vec<Quote> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let rt = Runtime::new().unwrap();
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let pool = Arc::new(Semaphore::new(MAX_TASK));
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<Quote>();
|
||||
|
||||
for page in 1..20 {
|
||||
let pool = Arc::clone(&pool);
|
||||
let tx = tx.clone();
|
||||
rt.spawn(async move {
|
||||
tokio::spawn(async move {
|
||||
let _permit = pool.acquire().await.unwrap();
|
||||
let text = download_quote_html(page).await.unwrap();
|
||||
let html = Html::parse_document(&text);
|
||||
@ -74,7 +72,7 @@ fn main() {
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
while let Some(quote) = rx.blocking_recv() {
|
||||
while let Some(quote) = rx.recv().await {
|
||||
println!("{:?}", quote);
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user