Upgrade dependencies and enable tokio macros

This commit is contained in:
hr567 2022-07-04 22:28:23 +08:00
parent 301cb4da31
commit f6298eef21
No known key found for this signature in database
GPG Key ID: 44CA67290D5250EA
3 changed files with 459 additions and 374 deletions

809
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,12 +1,12 @@
[package]
name = "quote-scrape"
name = "quote-scraper"
version = "0.1.0"
authors = ["hr567 <hr567@hr567.me>"]
edition = "2018"
[dependencies]
lazy_static = "1.4.0"
reqwest = "0.11.0"
scraper = "0.12.0"
tokio = { version = "1.0.2", features = [ "rt-multi-thread", "sync" ] }
url = "2.2.0"
reqwest = "0.11.11"
scraper = "0.13.0"
tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread", "sync"] }
url = "2.2.2"

View File

@ -3,10 +3,7 @@ use std::sync::Arc;
use lazy_static::lazy_static;
use reqwest::Client;
use scraper::{Html, Selector};
use tokio::{
runtime::Runtime,
sync::{mpsc, Semaphore},
};
use tokio::sync::{mpsc, Semaphore};
use url::Url;
const MAX_TASK: usize = 16;
@ -24,6 +21,7 @@ lazy_static! {
};
}
#[allow(dead_code)]
#[derive(Debug)]
struct Quote {
text: String,
@ -54,15 +52,15 @@ fn parse_quote_html(page: Html) -> Vec<Quote> {
.collect()
}
fn main() {
let rt = Runtime::new().unwrap();
#[tokio::main]
async fn main() {
let pool = Arc::new(Semaphore::new(MAX_TASK));
let (tx, mut rx) = mpsc::unbounded_channel::<Quote>();
for page in 1..20 {
let pool = Arc::clone(&pool);
let tx = tx.clone();
rt.spawn(async move {
tokio::spawn(async move {
let _permit = pool.acquire().await.unwrap();
let text = download_quote_html(page).await.unwrap();
let html = Html::parse_document(&text);
@ -74,7 +72,7 @@ fn main() {
}
drop(tx);
while let Some(quote) = rx.blocking_recv() {
while let Some(quote) = rx.recv().await {
println!("{:?}", quote);
}
}