hr567/rust-quote-scraper
1
0
Fork 0
Code Issues Pull Requests Activity

Upgrade dependencies and enable tokio macros

Browse Source
This commit is contained in:
hr567 2022-07-04 22:28:23 +08:00
parent 301cb4da31
commit f6298eef21
No known key found for this signature in database
GPG Key ID: 44CA67290D5250EA
3 changed files with 459 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

809
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

10
Cargo.toml
View File

@ -1,12 +1,12 @@
[package] [package]
name = "quote-scrape" name = "quote-scraper"
version = "0.1.0" version = "0.1.0"
authors = ["hr567 <hr567@hr567.me>"] authors = ["hr567 <hr567@hr567.me>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
lazy_static = "1.4.0" lazy_static = "1.4.0"
reqwest = "0.11.0" reqwest = "0.11.11"
scraper = "0.12.0" scraper = "0.13.0"
tokio = { version = "1.0.2", features = [ "rt-multi-thread", "sync" ] } tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread", "sync"] }
url = "2.2.0" url = "2.2.2"

14
src/main.rs
View File

@ -3,10 +3,7 @@ use std::sync::Arc;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use reqwest::Client; use reqwest::Client;
use scraper::{Html, Selector}; use scraper::{Html, Selector};
use tokio::{ use tokio::sync::{mpsc, Semaphore};
runtime::Runtime,
sync::{mpsc, Semaphore},
};
use url::Url; use url::Url;
const MAX_TASK: usize = 16; const MAX_TASK: usize = 16;
@ -24,6 +21,7 @@ lazy_static! {
}; };
} }
#[allow(dead_code)]
#[derive(Debug)] #[derive(Debug)]
struct Quote { struct Quote {
text: String, text: String,
@ -54,15 +52,15 @@ fn parse_quote_html(page: Html) -> Vec<Quote> {
.collect() .collect()
} }
fn main() { #[tokio::main]
let rt = Runtime::new().unwrap(); async fn main() {
let pool = Arc::new(Semaphore::new(MAX_TASK)); let pool = Arc::new(Semaphore::new(MAX_TASK));
let (tx, mut rx) = mpsc::unbounded_channel::<Quote>(); let (tx, mut rx) = mpsc::unbounded_channel::<Quote>();
for page in 1..20 { for page in 1..20 {
let pool = Arc::clone(&pool); let pool = Arc::clone(&pool);
let tx = tx.clone(); let tx = tx.clone();
rt.spawn(async move { tokio::spawn(async move {
let _permit = pool.acquire().await.unwrap(); let _permit = pool.acquire().await.unwrap();
let text = download_quote_html(page).await.unwrap(); let text = download_quote_html(page).await.unwrap();
let html = Html::parse_document(&text); let html = Html::parse_document(&text);
@ -74,7 +72,7 @@ fn main() {
} }
drop(tx); drop(tx);
while let Some(quote) = rx.blocking_recv() { while let Some(quote) = rx.recv().await {
println!("{:?}", quote); println!("{:?}", quote);
} }
} }