Compare commits
No commits in common. "774f3195deb3a768790bc234c57e43640b7d08ff" and "69e44b9cbea29601bdb6956fe5f008a5e16318e7" have entirely different histories.
774f3195de
...
69e44b9cbe
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -1,3 +1,2 @@
|
||||||
/target
|
/target
|
||||||
/data.pdf
|
data.pdf
|
||||||
/result
|
|
||||||
|
|
37
Cargo.lock
generated
37
Cargo.lock
generated
|
@ -288,17 +288,6 @@ version = "0.10.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
|
checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "atty"
|
|
||||||
version = "0.2.14"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
|
||||||
dependencies = [
|
|
||||||
"hermit-abi",
|
|
||||||
"libc",
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "autocfg"
|
name = "autocfg"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
@ -436,17 +425,6 @@ dependencies = [
|
||||||
"generic-array",
|
"generic-array",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "colored"
|
|
||||||
version = "2.0.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd"
|
|
||||||
dependencies = [
|
|
||||||
"atty",
|
|
||||||
"lazy_static",
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "convert_case"
|
name = "convert_case"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
|
@ -1411,12 +1389,10 @@ dependencies = [
|
||||||
"futures",
|
"futures",
|
||||||
"json",
|
"json",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
|
||||||
"pdf",
|
"pdf",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"scraper",
|
"scraper",
|
||||||
"simple_logger",
|
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"time 0.3.9",
|
"time 0.3.9",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
@ -1788,19 +1764,6 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "simple_logger"
|
|
||||||
version = "2.1.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c75a9723083573ace81ad0cdfc50b858aa3c366c48636edb4109d73122a0c0ea"
|
|
||||||
dependencies = [
|
|
||||||
"atty",
|
|
||||||
"colored",
|
|
||||||
"log",
|
|
||||||
"time 0.3.9",
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "siphasher"
|
name = "siphasher"
|
||||||
version = "0.3.10"
|
version = "0.3.10"
|
||||||
|
|
|
@ -14,12 +14,10 @@ bytes = "1.1"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
json = "0.12.4"
|
json = "0.12.4"
|
||||||
lazy_static = "1.4"
|
lazy_static = "1.4"
|
||||||
log = "0.4.16"
|
|
||||||
pdf = "0.7.2"
|
pdf = "0.7.2"
|
||||||
regex = "1.5.5"
|
regex = "1.5.5"
|
||||||
reqwest = { version = "0.11", features = ["rustls-tls", "stream"], default-features = false }
|
reqwest = { version = "0.11", features = ["rustls-tls", "stream"], default-features = false }
|
||||||
scraper = "0.12"
|
scraper = "0.12"
|
||||||
simple_logger = { version = "2.1.0", features = ["stderr"] }
|
|
||||||
thiserror = "1"
|
thiserror = "1"
|
||||||
time = { version = "0.3.9", features = ["formatting", "macros", "parsing"] }
|
time = { version = "0.3.9", features = ["formatting", "macros", "parsing"] }
|
||||||
tokio = { version = "1", features = ["full"] }
|
tokio = { version = "1", features = ["full"] }
|
||||||
|
|
|
@ -2,7 +2,6 @@ use std::time::{Duration, Instant, SystemTime};
|
||||||
|
|
||||||
use futures::{sink::SinkExt, TryStreamExt};
|
use futures::{sink::SinkExt, TryStreamExt};
|
||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
use log::info;
|
|
||||||
use reqwest::Url;
|
use reqwest::Url;
|
||||||
use scraper::Selector;
|
use scraper::Selector;
|
||||||
use time::PrimitiveDateTime;
|
use time::PrimitiveDateTime;
|
||||||
|
@ -51,8 +50,6 @@ impl PdfFetcher {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn fetch(&mut self) -> Result<pdf::file::File<Vec<u8>>, Error> {
|
pub async fn fetch(&mut self) -> Result<pdf::file::File<Vec<u8>>, Error> {
|
||||||
info!("Fetching data PDF");
|
|
||||||
|
|
||||||
let cache_modtime = match tokio::fs::File::open(CACHED_PDF_PATH).await {
|
let cache_modtime = match tokio::fs::File::open(CACHED_PDF_PATH).await {
|
||||||
Ok(file) => Some(
|
Ok(file) => Some(
|
||||||
file.metadata()
|
file.metadata()
|
||||||
|
@ -68,7 +65,6 @@ impl PdfFetcher {
|
||||||
if let Some(instant) = self.last_checked {
|
if let Some(instant) = self.last_checked {
|
||||||
if now - instant < *MIN_CHECK_INTERVAL {
|
if now - instant < *MIN_CHECK_INTERVAL {
|
||||||
return if cache_modtime.is_some() {
|
return if cache_modtime.is_some() {
|
||||||
info!("Already checked origin recently, not rechecking");
|
|
||||||
self.cached_pdf()
|
self.cached_pdf()
|
||||||
} else {
|
} else {
|
||||||
Err(Error::TooSoon)
|
Err(Error::TooSoon)
|
||||||
|
@ -115,8 +111,6 @@ impl PdfFetcher {
|
||||||
.map_or(true, |(cache, origin)| origin > cache);
|
.map_or(true, |(cache, origin)| origin > cache);
|
||||||
|
|
||||||
if outdated {
|
if outdated {
|
||||||
info!("Cached PDF is outdated, downloading latest version");
|
|
||||||
|
|
||||||
let mut pdf_stream = self
|
let mut pdf_stream = self
|
||||||
.client
|
.client
|
||||||
.get(pdf_url)
|
.get(pdf_url)
|
||||||
|
@ -130,8 +124,6 @@ impl PdfFetcher {
|
||||||
sink.send_all(&mut pdf_stream).await?;
|
sink.send_all(&mut pdf_stream).await?;
|
||||||
<dyn futures::Sink<bytes::Bytes, Error = std::io::Error> + Unpin>::close(&mut sink)
|
<dyn futures::Sink<bytes::Bytes, Error = std::io::Error> + Unpin>::close(&mut sink)
|
||||||
.await?;
|
.await?;
|
||||||
} else {
|
|
||||||
info!("Cached PDF is already up to date");
|
|
||||||
}
|
}
|
||||||
self.last_checked = Some(now);
|
self.last_checked = Some(now);
|
||||||
self.cached_pdf()
|
self.cached_pdf()
|
||||||
|
|
10
src/main.rs
10
src/main.rs
|
@ -1,9 +1,6 @@
|
||||||
use std::{sync::Arc, time::Duration};
|
use std::{sync::Arc, time::Duration};
|
||||||
|
|
||||||
use actix_web::{
|
use actix_web::{get, http::header::ContentType, web, App, HttpResponse, HttpServer, Responder};
|
||||||
get, http::header::ContentType, middleware::Logger, web, App, HttpResponse, HttpServer,
|
|
||||||
Responder,
|
|
||||||
};
|
|
||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
@ -50,10 +47,10 @@ async fn start_updater() -> Result<web::Data<AppState>, Error> {
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
actix_web::rt::System::new().block_on(async {
|
actix_web::rt::System::new().block_on(async {
|
||||||
loop {
|
loop {
|
||||||
actix_web::rt::time::sleep(*UPDATE_INTERVAL).await;
|
|
||||||
if let Err(e) = try_update(&state_copy, &mut fetcher).await {
|
if let Err(e) = try_update(&state_copy, &mut fetcher).await {
|
||||||
eprintln!("Error updating data: {:#?}", e);
|
eprintln!("Error updating data: {:#?}", e);
|
||||||
}
|
}
|
||||||
|
actix_web::rt::time::sleep(*UPDATE_INTERVAL).await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -63,14 +60,11 @@ async fn start_updater() -> Result<web::Data<AppState>, Error> {
|
||||||
|
|
||||||
#[actix_web::main]
|
#[actix_web::main]
|
||||||
async fn main() -> std::io::Result<()> {
|
async fn main() -> std::io::Result<()> {
|
||||||
simple_logger::init_with_level(log::Level::Info).unwrap();
|
|
||||||
|
|
||||||
let state = start_updater().await.expect("Failed to initialize state");
|
let state = start_updater().await.expect("Failed to initialize state");
|
||||||
|
|
||||||
HttpServer::new(move || {
|
HttpServer::new(move || {
|
||||||
App::new()
|
App::new()
|
||||||
.app_data(state.clone())
|
.app_data(state.clone())
|
||||||
.wrap(Logger::default())
|
|
||||||
.service(csv)
|
.service(csv)
|
||||||
.service(json)
|
.service(json)
|
||||||
.service(actix_files::Files::new("/", "./static/").index_file("index.html"))
|
.service(actix_files::Files::new("/", "./static/").index_file("index.html"))
|
||||||
|
|
Loading…
Reference in a new issue