xenofem/transbeam
1
0
Fork 0
Code Issues 6 Pull requests Projects Releases Wiki Activity

implement downloads, lots of misc tweaks/fixes

Browse source
This commit is contained in:
xenofem 2022-04-27 20:15:51 -04:00
parent 18caa2baf1
commit 2ec295e606
7 changed files with 354 additions and 854 deletions
Download patch file Download diff file Expand all files Collapse all files

79
src/zip.rs
View file

@ -8,30 +8,30 @@ use time::OffsetDateTime;
use crate::file::LiveWriter;
use crate::UploadedFile;
const SIGNATURE_SIZE: usize = 4;
const SHARED_FIELDS_SIZE: usize = 26;
const EXTRA_FIELD_SIZE: usize = 41;
const LOCAL_HEADER_SIZE_MINUS_FILENAME: usize =
const SIGNATURE_SIZE: u64 = 4;
const SHARED_FIELDS_SIZE: u64 = 26;
const EXTRA_FIELD_SIZE: u64 = 41;
const LOCAL_HEADER_SIZE_MINUS_FILENAME: u64 =
SIGNATURE_SIZE + SHARED_FIELDS_SIZE + EXTRA_FIELD_SIZE;
const DATA_DESCRIPTOR_SIZE: usize = 24;
const FILE_ENTRY_SIZE_MINUS_FILENAME_AND_FILE: usize =
const DATA_DESCRIPTOR_SIZE: u64 = 24;
const FILE_ENTRY_SIZE_MINUS_FILENAME_AND_FILE: u64 =
LOCAL_HEADER_SIZE_MINUS_FILENAME + DATA_DESCRIPTOR_SIZE;
const CENTRAL_DIRECTORY_HEADER_SIZE_MINUS_FILENAME: usize =
const CENTRAL_DIRECTORY_HEADER_SIZE_MINUS_FILENAME: u64 =
SIGNATURE_SIZE + 2 + SHARED_FIELDS_SIZE + 14 + EXTRA_FIELD_SIZE;
const EOCD64_RECORD_SIZE: usize = 56;
const EOCD64_LOCATOR_SIZE: usize = 20;
const EOCD_RECORD_SIZE: usize = 22;
const EOCD_TOTAL_SIZE: usize = EOCD64_RECORD_SIZE + EOCD64_LOCATOR_SIZE + EOCD_RECORD_SIZE;
const EOCD64_RECORD_SIZE: u64 = 56;
const EOCD64_LOCATOR_SIZE: u64 = 20;
const EOCD_RECORD_SIZE: u64 = 22;
const EOCD_TOTAL_SIZE: u64 = EOCD64_RECORD_SIZE + EOCD64_LOCATOR_SIZE + EOCD_RECORD_SIZE;
const EMPTY_STRING_CRC32: u32 = 0;
fn file_entry_size(file: &UploadedFile) -> usize {
FILE_ENTRY_SIZE_MINUS_FILENAME_AND_FILE + file.name.len() + file.size
fn file_entry_size(file: &UploadedFile) -> u64 {
FILE_ENTRY_SIZE_MINUS_FILENAME_AND_FILE + file.name.len() as u64 + file.size
}
fn file_entries_size(files: &[UploadedFile]) -> usize {
fn file_entries_size(files: &[UploadedFile]) -> u64 {
let mut total = 0;
for file in files.iter() {
total += file_entry_size(file)
@ -39,15 +39,15 @@ fn file_entries_size(files: &[UploadedFile]) -> usize {
total
}
fn central_directory_size(files: &[UploadedFile]) -> usize {
fn central_directory_size(files: &[UploadedFile]) -> u64 {
let mut total = 0;
for file in files.iter() {
total += CENTRAL_DIRECTORY_HEADER_SIZE_MINUS_FILENAME + file.name.len();
total += CENTRAL_DIRECTORY_HEADER_SIZE_MINUS_FILENAME + file.name.len() as u64;
}
total
}
fn zipfile_size(files: &[UploadedFile]) -> usize {
fn zipfile_size(files: &[UploadedFile]) -> u64 {
file_entries_size(files) + central_directory_size(files) + EOCD_TOTAL_SIZE
}
@ -102,7 +102,7 @@ impl UploadedFile {
fields
}
fn extra_field(&self, local_header_offset: usize) -> Vec<u8> {
fn extra_field(&self, local_header_offset: u64) -> Vec<u8> {
let mut field = vec![
0x01, 0x00, // Zip64 extended information
28, 0, // 28 bytes of data
@ -111,9 +111,9 @@ impl UploadedFile {
// header, we're supposed to leave these blank and point to
// the data descriptor, but I'm assuming it won't hurt to fill
// them in regardless
append_value(&mut field, self.size as u64, 8);
append_value(&mut field, self.size as u64, 8);
append_value(&mut field, local_header_offset as u64, 8);
append_value(&mut field, self.size, 8);
append_value(&mut field, self.size, 8);
append_value(&mut field, local_header_offset, 8);
append_0(&mut field, 4); // File starts on disk 0, there's no other disk
field.append(&mut vec![
@ -126,7 +126,7 @@ impl UploadedFile {
field
}
fn local_header(&self, local_header_offset: usize) -> Vec<u8> {
fn local_header(&self, local_header_offset: u64) -> Vec<u8> {
let mut header = vec![0x50, 0x4b, 0x03, 0x04]; // Local file header signature
header.append(&mut self.shared_header_fields(None));
header.append(&mut self.name.clone().into_bytes());
@ -134,7 +134,7 @@ impl UploadedFile {
header
}
fn central_directory_header(&self, local_header_offset: usize, hash: u32) -> Vec<u8> {
fn central_directory_header(&self, local_header_offset: u64, hash: u32) -> Vec<u8> {
let mut header = vec![
0x50, 0x4b, 0x01, 0x02, // Central directory file header signature
45, 3, // Made by a Unix system supporting version 4.5
@ -152,15 +152,15 @@ impl UploadedFile {
let mut descriptor = vec![0x50, 0x4b, 0x07, 0x08]; // Data descriptor signature
append_value(&mut descriptor, hash as u64, 4);
// Compressed and uncompressed sizes
append_value(&mut descriptor, self.size as u64, 8);
append_value(&mut descriptor, self.size as u64, 8);
append_value(&mut descriptor, self.size, 8);
append_value(&mut descriptor, self.size, 8);
descriptor
}
}
fn end_of_central_directory(files: &[UploadedFile]) -> Vec<u8> {
let entries_size = file_entries_size(files) as u64;
let directory_size = central_directory_size(files) as u64;
let entries_size = file_entries_size(files);
let directory_size = central_directory_size(files);
let mut eocd = vec![
0x50, 0x4b, 0x06, 0x06, // EOCD64 record signature
@ -190,18 +190,18 @@ fn end_of_central_directory(files: &[UploadedFile]) -> Vec<u8> {
eocd
}
pub struct ZipGenerator<'a> {
pub struct ZipGenerator<W: Write> {
files: Vec<UploadedFile>,
file_index: usize,
byte_index: usize,
byte_index: u64,
pending_metadata: Vec<u8>,
hasher: Hasher,
hashes: Vec<u32>,
output: Box<dyn LiveWriter + 'a>,
output: W,
}
impl<'a> ZipGenerator<'a> {
pub fn new(files: Vec<UploadedFile>, output: Box<dyn LiveWriter + 'a>) -> Self {
impl<W: Write> ZipGenerator<W> {
pub fn new(files: Vec<UploadedFile>, output: W) -> Self {
let mut result = Self {
files,
file_index: 0,
@ -215,7 +215,7 @@ impl<'a> ZipGenerator<'a> {
result
}
pub fn total_size(&self) -> usize {
pub fn total_size(&self) -> u64 {
zipfile_size(&self.files)
}
@ -225,7 +225,7 @@ impl<'a> ZipGenerator<'a> {
self.pending_metadata
.append(&mut self.files[self.file_index].data_descriptor(hash));
debug!(
"Finishing file entry in zipfile: {}, hash {}",
"Finishing file entry in zipfile: {}, hash {:x}",
self.files[self.file_index].name, hash
);
self.file_index += 1;
@ -243,7 +243,7 @@ impl<'a> ZipGenerator<'a> {
let mut local_header = self.files[self.file_index].local_header(offset);
let mut data_descriptor =
self.files[self.file_index].data_descriptor(EMPTY_STRING_CRC32);
offset += local_header.len() + data_descriptor.len();
offset += local_header.len() as u64 + data_descriptor.len() as u64;
self.file_index += 1;
self.pending_metadata.append(&mut local_header);
self.pending_metadata.append(&mut data_descriptor);
@ -279,13 +279,13 @@ impl<'a> ZipGenerator<'a> {
}
}
impl<'a> LiveWriter for ZipGenerator<'a> {
impl<W: LiveWriter> LiveWriter for ZipGenerator<W> {
fn add_waker(&mut self, waker: Waker) {
self.output.add_waker(waker);
}
}
impl<'a> Write for ZipGenerator<'a> {
impl<W: Write> Write for ZipGenerator<W> {
fn write(&mut self, mut buf: &[u8]) -> std::io::Result<usize> {
while !self.pending_metadata.is_empty() {
let result = self.output.write(self.pending_metadata.as_slice());
@ -302,14 +302,15 @@ impl<'a> Write for ZipGenerator<'a> {
return Ok(0);
}
let bytes_remaining = self.files[self.file_index].size - self.byte_index;
if bytes_remaining < buf.len() {
buf = &buf[..bytes_remaining];
if bytes_remaining < (buf.len() as u64) {
buf = &buf[..bytes_remaining as usize];
}
let result = self.output.write(buf);
match result {
Ok(0) | Err(_) => (),
Ok(n) => {
self.hasher.update(&buf[..n]);
let n = n as u64;
self.byte_index += n;
if n == bytes_remaining {
self.finish_file();