Compare commits
No commits in common. "2c7484b538cde623e6563572fb011f0aad861f13" and "d2eacd6a123483a227f7357df01d4aebc1851ef2" have entirely different histories.
2c7484b538
...
d2eacd6a12
5 changed files with 901 additions and 1289 deletions
2133
Cargo.lock
generated
2133
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
10
Cargo.toml
10
Cargo.toml
|
@ -14,12 +14,12 @@ bytes = "1.1"
|
|||
futures = "0.3"
|
||||
lazy_static = "1.4"
|
||||
log = "0.4.16"
|
||||
pdf = "0.9"
|
||||
pdf = "0.8"
|
||||
regex = "1.5.5"
|
||||
reqwest = { version = "0.12.23", features = ["rustls-tls", "stream"], default-features = false }
|
||||
scraper = "0.24"
|
||||
simple_logger = { version = "5", features = ["stderr"] }
|
||||
thiserror = "2"
|
||||
reqwest = { version = "0.11", features = ["rustls-tls", "stream"], default-features = false }
|
||||
scraper = "0.12"
|
||||
simple_logger = { version = "4.2", features = ["stderr"] }
|
||||
thiserror = "1"
|
||||
time = { version = "0.3.9", features = ["formatting", "macros", "parsing"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = "0.1.8"
|
||||
|
|
30
flake.lock
generated
30
flake.lock
generated
|
@ -6,11 +6,11 @@
|
|||
"rust-analyzer-src": "rust-analyzer-src"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1758695884,
|
||||
"narHash": "sha256-rnHjtBRkcwRkrUZxg0RqN1qWTG+QC/gj4vn9uzEkBww=",
|
||||
"lastModified": 1721975407,
|
||||
"narHash": "sha256-XkNqglPxkfWOkysN5C1aZeoHCozwyRq3nC4jL0IFqlA=",
|
||||
"owner": "nix-community",
|
||||
"repo": "fenix",
|
||||
"rev": "9cdb79384d02234fb2868eba6c7d390253ef6f83",
|
||||
"rev": "27128b6e467ced6142264d02a884fde45931e708",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -21,11 +21,11 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1758427187,
|
||||
"narHash": "sha256-pHpxZ/IyCwoTQPtFIAG2QaxuSm8jWzrzBGjwQZIttJc=",
|
||||
"lastModified": 1721743106,
|
||||
"narHash": "sha256-adRZhFpBTnHiK3XIELA3IBaApz70HwCYfv7xNrHjebA=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "554be6495561ff07b6c724047bdd7e0716aa7b46",
|
||||
"rev": "dc14ed91132ee3a26255d01d8fd0c1f5bff27b2f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -37,11 +37,11 @@
|
|||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1758427187,
|
||||
"narHash": "sha256-pHpxZ/IyCwoTQPtFIAG2QaxuSm8jWzrzBGjwQZIttJc=",
|
||||
"lastModified": 1721924956,
|
||||
"narHash": "sha256-Sb1jlyRO+N8jBXEX9Pg9Z1Qb8Bw9QyOgLDNMEpmjZ2M=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "554be6495561ff07b6c724047bdd7e0716aa7b46",
|
||||
"rev": "5ad6a14c6bf098e98800b091668718c336effc95",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -61,11 +61,11 @@
|
|||
"rust-analyzer-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1758620797,
|
||||
"narHash": "sha256-Ly4rHgrixFMBnkbMursVt74mxnntnE6yVdF5QellJ+A=",
|
||||
"lastModified": 1721909430,
|
||||
"narHash": "sha256-u3e38jvjbxbbYH3caQFPE7gnNFNBjbkqkHRKBx1AOJs=",
|
||||
"owner": "rust-lang",
|
||||
"repo": "rust-analyzer",
|
||||
"rev": "905641f3520230ad6ef421bcf5da9c6b49f2479b",
|
||||
"rev": "c02a4a31eada45e591b5edb8c1a813ea7b9d408f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -95,11 +95,11 @@
|
|||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"lastModified": 1710146030,
|
||||
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use pdf::{backend::Backend, content::{Op, Point, TextDrawAdjusted}, file::{NoCache, NoLog}};
|
||||
use pdf::{backend::Backend, content::{Op, Point, TextDrawAdjusted}};
|
||||
use regex::Regex;
|
||||
use time::Date;
|
||||
|
||||
|
@ -28,7 +28,7 @@ pub enum Error {
|
|||
}
|
||||
|
||||
impl DataSet {
|
||||
pub fn extract<B: Backend>(doc: &pdf::file::File<B, NoCache, NoCache, NoLog>) -> Result<Self, Error> {
|
||||
pub fn extract<B: Backend>(doc: &pdf::file::File<B>) -> Result<Self, Error> {
|
||||
let mut doc_iter = DocumentIterator::new(doc).peekable();
|
||||
|
||||
let mut columns: Vec<(Arc<String>, f32)> = Vec::new();
|
||||
|
@ -113,13 +113,13 @@ struct DocumentIterator<'a> {
|
|||
}
|
||||
|
||||
impl<'a> DocumentIterator<'a> {
|
||||
fn new<B: Backend>(document: &'a pdf::file::File<B, NoCache, NoCache, NoLog>) -> Self {
|
||||
fn new<B: Backend>(document: &'a pdf::file::File<B>) -> Self {
|
||||
Self {
|
||||
point: Point { x: 0.0, y: 0.0 },
|
||||
operations: Box::new(
|
||||
document
|
||||
.pages()
|
||||
.filter_map(|page| Some(page.ok()?.contents.clone()?.operations(&document.resolver()).ok()?.into_iter()))
|
||||
.filter_map(|page| Some(page.ok()?.contents.clone()?.operations(document).ok()?.into_iter()))
|
||||
.flatten(),
|
||||
),
|
||||
}
|
||||
|
|
|
@ -6,7 +6,6 @@ use std::{
|
|||
use futures::{sink::SinkExt, TryStreamExt};
|
||||
use lazy_static::lazy_static;
|
||||
use log::{info, warn};
|
||||
use pdf::file::{NoCache, NoLog};
|
||||
use reqwest::Url;
|
||||
use scraper::Selector;
|
||||
use time::PrimitiveDateTime;
|
||||
|
@ -14,7 +13,7 @@ use tokio_util::codec;
|
|||
|
||||
lazy_static! {
|
||||
static ref CHARTS_URL: Url = Url::parse("https://www.mwra.com/biobot/biobotdata.htm").unwrap();
|
||||
static ref PDF_SEL: Selector = Selector::parse(r#"a[href^="/media/file/mwradata"][href$="-datapdf"]"#).unwrap();
|
||||
static ref PDF_SEL: Selector = Selector::parse(r#"a[href^="/media/file/mwradata"][href$="-data"]"#).unwrap();
|
||||
static ref MIN_CHECK_INTERVAL: Duration = Duration::from_secs(300);
|
||||
}
|
||||
|
||||
|
@ -55,7 +54,7 @@ impl PdfFetcher {
|
|||
})
|
||||
}
|
||||
|
||||
pub async fn fetch(&mut self) -> Result<pdf::file::File<Vec<u8>, NoCache, NoCache, NoLog>, Error> {
|
||||
pub async fn fetch(&mut self) -> Result<pdf::file::File<Vec<u8>>, Error> {
|
||||
info!("Fetching data PDF");
|
||||
|
||||
let cache_modtime = match tokio::fs::File::open(&self.cached_pdf_path).await {
|
||||
|
@ -146,7 +145,7 @@ impl PdfFetcher {
|
|||
self.cached_pdf()
|
||||
}
|
||||
|
||||
fn cached_pdf(&self) -> Result<pdf::file::File<Vec<u8>, NoCache, NoCache, NoLog>, Error> {
|
||||
Ok(pdf::file::FileOptions::uncached().open(&self.cached_pdf_path)?)
|
||||
fn cached_pdf(&self) -> Result<pdf::file::File<Vec<u8>>, Error> {
|
||||
Ok(pdf::file::File::open(&self.cached_pdf_path)?)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue