Compare commits
	
		
			2 commits
		
	
	
		
			d7dfc28587
			...
			db24d0a497
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| db24d0a497 | |||
| 4eb510039c | 
					 2 changed files with 23 additions and 11 deletions
				
			
		
							
								
								
									
										23
									
								
								src/fetch.rs
									
										
									
									
									
								
							
							
						
						
									
										23
									
								
								src/fetch.rs
									
										
									
									
									
								
							| 
						 | 
				
			
			@ -1,8 +1,11 @@
 | 
			
		|||
use std::{time::{Duration, Instant, SystemTime}, path::PathBuf};
 | 
			
		||||
use std::{
 | 
			
		||||
    path::PathBuf,
 | 
			
		||||
    time::{Duration, Instant, SystemTime},
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
use futures::{sink::SinkExt, TryStreamExt};
 | 
			
		||||
use lazy_static::lazy_static;
 | 
			
		||||
use log::info;
 | 
			
		||||
use log::{info, warn};
 | 
			
		||||
use reqwest::Url;
 | 
			
		||||
use scraper::Selector;
 | 
			
		||||
use time::PrimitiveDateTime;
 | 
			
		||||
| 
						 | 
				
			
			@ -94,11 +97,9 @@ impl PdfFetcher {
 | 
			
		|||
            .ok_or(Error::NotFound)?;
 | 
			
		||||
        let pdf_url = CHARTS_URL.join(pdf_href)?;
 | 
			
		||||
 | 
			
		||||
        let origin_modtime = self
 | 
			
		||||
            .client
 | 
			
		||||
            .head(pdf_url.clone())
 | 
			
		||||
            .send()
 | 
			
		||||
            .await?
 | 
			
		||||
        let head_resp = self.client.head(pdf_url.clone()).send().await?;
 | 
			
		||||
 | 
			
		||||
        let origin_modtime = head_resp
 | 
			
		||||
            .headers()
 | 
			
		||||
            .get(reqwest::header::LAST_MODIFIED)
 | 
			
		||||
            .and_then(|val| {
 | 
			
		||||
| 
						 | 
				
			
			@ -115,7 +116,13 @@ impl PdfFetcher {
 | 
			
		|||
            .zip(origin_modtime)
 | 
			
		||||
            .map_or(true, |(cache, origin)| origin > cache);
 | 
			
		||||
 | 
			
		||||
        if outdated {
 | 
			
		||||
        if !head_resp.status().is_success() {
 | 
			
		||||
            warn!(
 | 
			
		||||
                "MWRA server returned unexpected response, not fetching updated PDF: {:?}",
 | 
			
		||||
                head_resp
 | 
			
		||||
            );
 | 
			
		||||
            // Just use the cached PDF, assuming we have it
 | 
			
		||||
        } else if outdated {
 | 
			
		||||
            info!("Cached PDF is outdated, downloading latest version");
 | 
			
		||||
 | 
			
		||||
            let mut pdf_stream = self
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										11
									
								
								src/main.rs
									
										
									
									
									
								
							
							
						
						
									
										11
									
								
								src/main.rs
									
										
									
									
									
								
							| 
						 | 
				
			
			@ -41,7 +41,9 @@ async fn try_update(state: &AppState, fetcher: &mut PdfFetcher) -> Result<(), Er
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
async fn start_updater() -> Result<web::Data<AppState>, Error> {
 | 
			
		||||
    let cached_pdf_path = PathBuf::from(std::env::var("CACHED_PDF_PATH").unwrap_or_else(|_| String::from("data.pdf")));
 | 
			
		||||
    let cached_pdf_path = PathBuf::from(
 | 
			
		||||
        std::env::var("CACHED_PDF_PATH").unwrap_or_else(|_| String::from("data.pdf")),
 | 
			
		||||
    );
 | 
			
		||||
    let mut fetcher = PdfFetcher::new(cached_pdf_path)?;
 | 
			
		||||
    let state = web::Data::new(AppState {
 | 
			
		||||
        dataset: RwLock::new(Arc::new(load_data(&mut fetcher).await?)),
 | 
			
		||||
| 
						 | 
				
			
			@ -66,14 +68,17 @@ async fn start_updater() -> Result<web::Data<AppState>, Error> {
 | 
			
		|||
async fn main() -> std::io::Result<()> {
 | 
			
		||||
    simple_logger::init_with_level(log::Level::Info).unwrap();
 | 
			
		||||
 | 
			
		||||
    let static_dir = PathBuf::from(std::env::var("STATIC_DIR").unwrap_or_else(|_| String::from("static")));
 | 
			
		||||
    let static_dir =
 | 
			
		||||
        PathBuf::from(std::env::var("STATIC_DIR").unwrap_or_else(|_| String::from("static")));
 | 
			
		||||
 | 
			
		||||
    let state = start_updater().await.expect("Failed to initialize state");
 | 
			
		||||
 | 
			
		||||
    HttpServer::new(move || {
 | 
			
		||||
        App::new()
 | 
			
		||||
            .app_data(state.clone())
 | 
			
		||||
            .wrap(Logger::new(r#"%{r}a "%r" %s %b "%{Referer}i" "%{User-Agent}i" %T"#))
 | 
			
		||||
            .wrap(Logger::new(
 | 
			
		||||
                r#"%{r}a "%r" %s %b "%{Referer}i" "%{User-Agent}i" %T"#,
 | 
			
		||||
            ))
 | 
			
		||||
            .service(csv)
 | 
			
		||||
            .service(json)
 | 
			
		||||
            .service(actix_files::Files::new("/", static_dir.clone()).index_file("index.html"))
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue