9 changed files with 639 additions and 807 deletions
-
2Cargo.lock
-
2Cargo.toml
-
675src/client.rs
-
16src/download_error.rs
-
193src/m3u8_download.rs
-
57src/main.rs
-
462src/new_client.rs
-
37src/new_download_error.rs
-
2src/process.rs
@ -1,45 +1,204 @@ |
|||||
use std::{ffi::OsString, path::Path};
|
|
||||
|
use std::path::{Path, PathBuf};
|
||||
|
|
||||
use anyhow::anyhow;
|
use anyhow::anyhow;
|
||||
|
use futures_util::future::join_all;
|
||||
use http::{uri::{Authority, Scheme}, Uri};
|
use http::{uri::{Authority, Scheme}, Uri};
|
||||
|
use log::debug;
|
||||
|
use m3u8_rs::{MediaPlaylist, MediaSegment, Playlist};
|
||||
|
use tokio::{io::AsyncWriteExt as _, fs::File};
|
||||
|
|
||||
|
use crate::client::{ClientActorHandle, DownloadState};
|
||||
|
|
||||
#[derive(Debug)]
|
|
||||
pub(super) enum DownloadState {
|
|
||||
Open, // Nothing is done.
|
|
||||
Prepared, // The Uris to all .ts downloads are prepared.
|
|
||||
|
|
||||
|
#[derive(Clone, Debug)]
|
||||
|
pub(super) enum TsState {
|
||||
|
Created, // Nothing is done.
|
||||
|
Failed, // The download has failed.
|
||||
Ready, // All .ts downloads are done.
|
Ready, // All .ts downloads are done.
|
||||
}
|
}
|
||||
|
|
||||
|
#[derive(Clone, Debug)]
|
||||
struct TsPart {
|
struct TsPart {
|
||||
filename: OsString,
|
|
||||
url: Uri,
|
|
||||
state: DownloadState,
|
|
||||
|
#[allow(dead_code)]
|
||||
|
filename: PathBuf,
|
||||
|
#[allow(dead_code)]
|
||||
|
uri: Uri,
|
||||
|
#[allow(dead_code)]
|
||||
|
state: TsState,
|
||||
|
content_type: Option<String>,
|
||||
}
|
}
|
||||
|
|
||||
struct M3u8Download {
|
|
||||
|
#[derive(Debug)]
|
||||
|
pub(super) struct M3u8Download {
|
||||
|
#[allow(dead_code)]
|
||||
|
index_uri: Uri,
|
||||
|
#[allow(dead_code)]
|
||||
scheme: Scheme,
|
scheme: Scheme,
|
||||
|
#[allow(dead_code)]
|
||||
auth: Authority,
|
auth: Authority,
|
||||
|
#[allow(dead_code)]
|
||||
base_path: String,
|
base_path: String,
|
||||
|
#[allow(dead_code)]
|
||||
ts_parts: Vec<TsPart>,
|
ts_parts: Vec<TsPart>,
|
||||
}
|
}
|
||||
|
|
||||
|
|
||||
|
impl TsPart {
|
||||
|
fn new(uri: Uri) -> Self {
|
||||
|
let filename = PathBuf::from(Path::new(uri.path())
|
||||
|
. file_name()
|
||||
|
. expect("no filename in path_and_query"));
|
||||
|
let state = TsState::Created;
|
||||
|
let content_type = None;
|
||||
|
|
||||
|
Self { filename, uri, state, content_type }
|
||||
|
}
|
||||
|
|
||||
|
async fn download(&mut self, client: &ClientActorHandle) -> &Self {
|
||||
|
let state = client.download(self.filename.clone(), &self.uri).await;
|
||||
|
match state {
|
||||
|
Ok(Some(DownloadState::Done { content_type })) => {
|
||||
|
self.state = TsState::Ready;
|
||||
|
self.content_type = content_type;
|
||||
|
},
|
||||
|
_ => self.state = TsState::Failed,
|
||||
|
};
|
||||
|
|
||||
|
self
|
||||
|
}
|
||||
|
}
|
||||
|
|
||||
impl M3u8Download {
|
impl M3u8Download {
|
||||
pub(super) fn new(uri: Uri) -> anyhow::Result<Self> {
|
|
||||
let scheme = uri.scheme()
|
|
||||
|
pub(super) async fn new(m3u8_data: Vec<u8>, index_uri: Uri) -> anyhow::Result<Self> {
|
||||
|
let scheme = index_uri.scheme()
|
||||
. ok_or(anyhow!("Problem scheme in m3u8 uri"))?
|
. ok_or(anyhow!("Problem scheme in m3u8 uri"))?
|
||||
. clone();
|
|
||||
let auth= uri.authority()
|
|
||||
|
. to_owned();
|
||||
|
let auth = index_uri.authority()
|
||||
. ok_or(anyhow!("Problem authority in m3u8 uri"))?
|
. ok_or(anyhow!("Problem authority in m3u8 uri"))?
|
||||
. clone();
|
|
||||
let base_path = Path::new(uri.path()).parent()
|
|
||||
|
. to_owned();
|
||||
|
let base_path = Path::new(index_uri.path()).parent()
|
||||
. ok_or(anyhow!("Path problem"))?
|
. ok_or(anyhow!("Path problem"))?
|
||||
. to_str()
|
. to_str()
|
||||
. ok_or(anyhow!("Path problem"))?
|
. ok_or(anyhow!("Path problem"))?
|
||||
. to_string();
|
. to_string();
|
||||
let ts_parts = vec![];
|
|
||||
|
|
||||
Ok(Self {scheme, auth, base_path, ts_parts})
|
|
||||
|
let mut ts_parts = vec![];
|
||||
|
|
||||
|
match m3u8_rs::parse_playlist(&m3u8_data) {
|
||||
|
Result::Err(e) => Err(anyhow!("m3u8 parse error: {}", e))?,
|
||||
|
|
||||
|
Result::Ok((_, Playlist::MasterPlaylist(_))) =>
|
||||
|
Err(anyhow!("Master playlist not supported now"))?,
|
||||
|
|
||||
|
Result::Ok((_, Playlist::MediaPlaylist(pl))) => {
|
||||
|
Self::write_playlist(&index_uri, &pl).await?;
|
||||
|
for segment in pl.segments.iter() {
|
||||
|
let uri = Self::download_uri(&scheme, &auth, &base_path, segment)?;
|
||||
|
ts_parts.push(TsPart::new(uri));
|
||||
|
}
|
||||
|
},
|
||||
|
};
|
||||
|
|
||||
|
Ok(Self {index_uri, scheme, auth, base_path, ts_parts})
|
||||
|
}
|
||||
|
|
||||
|
pub(super) fn index_uri(&self) -> &Uri {
|
||||
|
&self.index_uri
|
||||
|
}
|
||||
|
|
||||
|
pub(super) async fn download(&mut self, client: &ClientActorHandle) {
|
||||
|
loop {
|
||||
|
let unfinished: Vec<_> = self.ts_parts.iter_mut()
|
||||
|
. filter_map(|p| match p.state {
|
||||
|
TsState::Ready => if p.content_type != Some("video/MP2T".to_string()) {
|
||||
|
Some(p.download(client))
|
||||
|
} else {
|
||||
|
None
|
||||
|
}
|
||||
|
|
||||
|
_ => Some(p.download(client))
|
||||
|
}).collect();
|
||||
|
|
||||
|
debug!("UNFINISHED NOW: {}", unfinished.len());
|
||||
|
|
||||
|
if unfinished.is_empty() { break; }
|
||||
|
|
||||
|
join_all(unfinished).await;
|
||||
|
}
|
||||
|
}
|
||||
|
|
||||
|
fn download_uri( scheme: &Scheme
|
||||
|
, auth: &Authority
|
||||
|
, base_path: &str |
||||
|
, segment: &MediaSegment ) -> anyhow::Result<Uri>
|
||||
|
{
|
||||
|
match Uri::try_from(segment.uri.clone()) {
|
||||
|
Ok(uri) => {
|
||||
|
let scheme = uri.scheme().unwrap_or(scheme);
|
||||
|
let auth = uri.authority().unwrap_or(auth);
|
||||
|
let path_and_query = uri.path_and_query()
|
||||
|
. ok_or(anyhow!("No path in Uri"))?;
|
||||
|
|
||||
|
Ok(Uri::builder() . scheme(scheme.clone())
|
||||
|
. authority(auth.clone())
|
||||
|
. path_and_query(path_and_query.clone())
|
||||
|
. build()?)
|
||||
|
}
|
||||
|
|
||||
|
Err(_) => {
|
||||
|
Ok(Uri::builder() . scheme(scheme.clone())
|
||||
|
. authority(auth.clone())
|
||||
|
. path_and_query(base_path.to_owned() + "/" + &segment.uri)
|
||||
|
. build()?)
|
||||
|
}
|
||||
|
}
|
||||
|
}
|
||||
|
|
||||
|
async fn write_playlist(uri: &Uri, playlist: &MediaPlaylist) -> anyhow::Result<()> {
|
||||
|
let filename = Path::new(uri.path())
|
||||
|
. file_name()
|
||||
|
. ok_or(anyhow!("can't extract filename from uri"))?;
|
||||
|
let mut file = File::create(filename).await?;
|
||||
|
|
||||
|
let segments: anyhow::Result<Vec<_>> = playlist.segments.iter().map(|s| {
|
||||
|
let mut new_segment = s.clone();
|
||||
|
Ok(match Uri::try_from(s.uri.clone()) {
|
||||
|
Ok(uri) => {
|
||||
|
new_segment.uri = Self::uri_relative_path(&uri)?;
|
||||
|
new_segment
|
||||
|
}
|
||||
|
|
||||
|
Err(_) => {
|
||||
|
let uri = s.uri.split_once('?')
|
||||
|
. map(|(s,_)| s)
|
||||
|
. unwrap_or(&s.uri);
|
||||
|
new_segment.uri = Self::relative_path(uri)?;
|
||||
|
new_segment
|
||||
|
}
|
||||
|
})
|
||||
|
}).collect();
|
||||
|
|
||||
|
let mut out_pl = playlist.clone();
|
||||
|
out_pl.segments = segments?;
|
||||
|
let mut file_data = vec![];
|
||||
|
out_pl.write_to(&mut file_data)?;
|
||||
|
file.write_all(&file_data).await?;
|
||||
|
|
||||
|
Ok(())
|
||||
|
}
|
||||
|
|
||||
|
fn uri_relative_path(uri: &Uri) -> anyhow::Result<String>
|
||||
|
{
|
||||
|
Self::relative_path(uri.path())
|
||||
|
}
|
||||
|
|
||||
|
fn relative_path(path: &str) -> anyhow::Result<String>
|
||||
|
{
|
||||
|
let filename = Path::new(path)
|
||||
|
. file_name()
|
||||
|
. ok_or(anyhow!("name error"))?
|
||||
|
. to_str()
|
||||
|
. ok_or(anyhow!("Error getting filename from uri"))?;
|
||||
|
Ok(filename.to_string())
|
||||
}
|
}
|
||||
}
|
}
|
||||
@ -1,462 +0,0 @@ |
|||||
use std::{collections::HashMap, io::ErrorKind, path::{Path, PathBuf}, time::Duration};
|
|
||||
|
|
||||
use anyhow::anyhow;
|
|
||||
use futures_util::StreamExt as _;
|
|
||||
use http::{
|
|
||||
header::{CONTENT_LENGTH, RANGE},
|
|
||||
request::Builder as RequestBuilder,
|
|
||||
HeaderMap,
|
|
||||
HeaderValue,
|
|
||||
Request,
|
|
||||
Response,
|
|
||||
StatusCode,
|
|
||||
Uri
|
|
||||
};
|
|
||||
use http_body_util::BodyDataStream;
|
|
||||
use reqwest::{redirect::Policy, Body};
|
|
||||
use tokio::{
|
|
||||
fs::{symlink_metadata, File},
|
|
||||
io::AsyncWriteExt as _,
|
|
||||
select,
|
|
||||
sync::{mpsc, oneshot},
|
|
||||
task::JoinSet,
|
|
||||
time::timeout
|
|
||||
};
|
|
||||
use tower::{util::BoxCloneService, ServiceBuilder, ServiceExt as _};
|
|
||||
use tower_http::decompression::{DecompressionBody, DecompressionLayer};
|
|
||||
use tower_http_client::{client::BodyReader, ServiceExt as _};
|
|
||||
use tower_reqwest::HttpClientLayer;
|
|
||||
|
|
||||
use crate::{
|
|
||||
new_download_error::DownloadError,
|
|
||||
m3u8_download::DownloadState
|
|
||||
};
|
|
||||
|
|
||||
use log::{debug, error, info};
|
|
||||
|
|
||||
|
|
||||
#[derive(Debug)]
|
|
||||
pub(super) enum ClientActorMessage {
|
|
||||
Download {
|
|
||||
filename: PathBuf,
|
|
||||
uri: Uri,
|
|
||||
respond_to: oneshot::Sender<DownloadState>,
|
|
||||
},
|
|
||||
GetData {
|
|
||||
uri: Uri,
|
|
||||
respond_to: oneshot::Sender<Option<Vec<u8>>>,
|
|
||||
},
|
|
||||
}
|
|
||||
|
|
||||
#[derive(Clone, Debug)]
|
|
||||
pub(super) enum ClientActorMessageHandle {
|
|
||||
Download {
|
|
||||
filename: PathBuf,
|
|
||||
uri: Uri,
|
|
||||
message: ActionIndex,
|
|
||||
},
|
|
||||
GetData {
|
|
||||
uri: Uri,
|
|
||||
buffer: Option<Vec<u8>>,
|
|
||||
message: ActionIndex,
|
|
||||
},
|
|
||||
}
|
|
||||
|
|
||||
pub(super) type ActionIndex = u64;
|
|
||||
type JoinSetResult = Result<Option<ClientActorMessageHandle>, DownloadError>;
|
|
||||
type HttpClient = BoxCloneService<Request<Body>, Response<DecompressionBody<Body>>, anyhow::Error>;
|
|
||||
|
|
||||
|
|
||||
#[derive(Debug)]
|
|
||||
struct ClientActor {
|
|
||||
timeout: Duration,
|
|
||||
client: HttpClient,
|
|
||||
tasks: JoinSet<JoinSetResult>,
|
|
||||
actions: HashMap<ActionIndex, ClientActorMessage>,
|
|
||||
actions_idx: ActionIndex,
|
|
||||
|
|
||||
receiver: mpsc::Receiver<ClientActorMessage>,
|
|
||||
}
|
|
||||
|
|
||||
|
|
||||
pub(super) struct ClientActorHandle {
|
|
||||
sender: mpsc::Sender<ClientActorMessage>,
|
|
||||
abort: oneshot::Sender<JoinSetResult>,
|
|
||||
}
|
|
||||
|
|
||||
async fn run_client(mut actor: ClientActor) {
|
|
||||
loop {
|
|
||||
select! {
|
|
||||
Some(join) = actor.tasks.join_next() => {
|
|
||||
match join {
|
|
||||
Err(e) => {
|
|
||||
error!("FATAL Join failed: {}", e);
|
|
||||
break
|
|
||||
},
|
|
||||
|
|
||||
Ok(Err(e)) => {
|
|
||||
info!("Retry failed download: {:?}", e);
|
|
||||
// retry ... instead of responing here we could also respond
|
|
||||
// with something that in turn would be used to retry...
|
|
||||
let client = actor.client.clone();
|
|
||||
actor.tasks.spawn(async move {
|
|
||||
download(client, e.action, actor.timeout).await
|
|
||||
});
|
|
||||
},
|
|
||||
|
|
||||
// when the task finishes
|
|
||||
Ok(Ok(Some(action))) => {
|
|
||||
info!("Done download: {:?}", action);
|
|
||||
use ClientActorMessageHandle::{Download, GetData};
|
|
||||
|
|
||||
match action {
|
|
||||
Download { filename: _, uri: _, ref message } => {
|
|
||||
if let Some((_, message)) = actor.actions.remove_entry(message) {
|
|
||||
use ClientActorMessage::Download;
|
|
||||
match message {
|
|
||||
Download { filename: _, uri: _, respond_to } => {
|
|
||||
let _ = respond_to.send(DownloadState::Ready);
|
|
||||
},
|
|
||||
|
|
||||
_ => panic!("Wrong variant ... this should never happen"),
|
|
||||
}
|
|
||||
} else {
|
|
||||
panic!("Lost a message");
|
|
||||
}
|
|
||||
},
|
|
||||
|
|
||||
GetData { uri: _, buffer, ref message } => {
|
|
||||
if let Some((_, message)) = actor.actions.remove_entry(message) {
|
|
||||
use ClientActorMessage::GetData;
|
|
||||
match message {
|
|
||||
GetData { uri: _, respond_to } => {
|
|
||||
let _ = respond_to.send(buffer);
|
|
||||
},
|
|
||||
|
|
||||
_ => panic!("Wrong variant ... this should never happen"),
|
|
||||
}
|
|
||||
} else {
|
|
||||
panic!("Lost a message");
|
|
||||
}
|
|
||||
},
|
|
||||
}
|
|
||||
},
|
|
||||
|
|
||||
// Got a stop message...here we still continue procession until the
|
|
||||
// JoinSet is empty.
|
|
||||
Ok(Ok(None)) => (),
|
|
||||
};
|
|
||||
}
|
|
||||
|
|
||||
Some(message) = actor.receiver.recv() => {
|
|
||||
actor.handle_message(message).await;
|
|
||||
}
|
|
||||
|
|
||||
else => {}
|
|
||||
}
|
|
||||
}
|
|
||||
}
|
|
||||
|
|
||||
async fn file_size(filename: &Path) -> u64 {
|
|
||||
// - get all informations to eventually existing file
|
|
||||
let metadata = match symlink_metadata(filename).await {
|
|
||||
Ok(metadata) => Some(metadata),
|
|
||||
Err(error) => match error.kind() {
|
|
||||
// If we can't write to a file we need to ... well theres nothing we can do
|
|
||||
ErrorKind::PermissionDenied => panic!("Permission denied on: {:?}", filename),
|
|
||||
_ => None,
|
|
||||
}
|
|
||||
};
|
|
||||
|
|
||||
metadata.map_or(0, |m| m.len())
|
|
||||
}
|
|
||||
|
|
||||
async fn request( client: &mut HttpClient
|
|
||||
, method: &str |
|
||||
, uri: &Uri
|
|
||||
, headers: HeaderMap ) -> anyhow::Result<Response<DecompressionBody<Body>>> {
|
|
||||
let mut request = RequestBuilder::new()
|
|
||||
. method(method)
|
|
||||
. uri(uri)
|
|
||||
. body(Body::default())?;
|
|
||||
|
|
||||
request.headers_mut().extend(headers);
|
|
||||
|
|
||||
debug!("Request: {:?}", request);
|
|
||||
|
|
||||
let response = client.execute(request).await?;
|
|
||||
|
|
||||
debug!("Response: {:?}", response.headers());
|
|
||||
|
|
||||
anyhow::ensure!( response.status().is_success()
|
|
||||
, "resonse status failed: {}"
|
|
||||
, response.status() );
|
|
||||
|
|
||||
Ok(response)
|
|
||||
}
|
|
||||
|
|
||||
async fn content_length( client: &mut HttpClient
|
|
||||
, uri: &Uri ) -> anyhow::Result<Option<u64>> {
|
|
||||
let head = request(client, "HEAD", uri, HeaderMap::new()).await?;
|
|
||||
|
|
||||
Ok(head . headers().get(CONTENT_LENGTH)
|
|
||||
. map(|v| v . to_str()
|
|
||||
. expect("unable to get CONTENT-LENGTH value")
|
|
||||
. parse::<u64>()
|
|
||||
. expect("unable to parse CONTENT-LENGTH value")))
|
|
||||
}
|
|
||||
|
|
||||
async fn open_outfile(status: &StatusCode, filename: &Path) -> File {
|
|
||||
match status {
|
|
||||
&StatusCode::PARTIAL_CONTENT =>
|
|
||||
// Here we assume that this response only comes if the requested
|
|
||||
// range was fullfillable and thus is the data range in the
|
|
||||
// response. Thats why I do not check the content-range header.
|
|
||||
// If that assumption does not hold this needs to be fixec.
|
|
||||
File::options() . create(true)
|
|
||||
. append(true)
|
|
||||
. open(filename)
|
|
||||
. await
|
|
||||
. expect("can not create file for writing"),
|
|
||||
|
|
||||
_ =>
|
|
||||
File::create(filename) . await
|
|
||||
. expect("can not create file for writing"),
|
|
||||
}
|
|
||||
}
|
|
||||
|
|
||||
async fn store_body( file: &mut File
|
|
||||
, body: &mut DecompressionBody<Body>
|
|
||||
, io_timeout: Duration ) -> anyhow::Result<()> {
|
|
||||
let mut body = BodyDataStream::new(body);
|
|
||||
|
|
||||
loop {
|
|
||||
// give timeout somehow... probably from client.
|
|
||||
let data = timeout(io_timeout, body.next()).await?;
|
|
||||
match data {
|
|
||||
None => break,
|
|
||||
Some(Err(e)) => {
|
|
||||
return Err(anyhow!(e.to_string()));
|
|
||||
}
|
|
||||
|
|
||||
Some(Ok(data)) => {
|
|
||||
file . write_all(data.as_ref()).await?;
|
|
||||
file . flush().await?;
|
|
||||
},
|
|
||||
}
|
|
||||
};
|
|
||||
|
|
||||
Ok(())
|
|
||||
}
|
|
||||
|
|
||||
async fn download( mut client: HttpClient
|
|
||||
, message: ClientActorMessageHandle
|
|
||||
, io_timeout: Duration ) -> JoinSetResult {
|
|
||||
use ClientActorMessageHandle::Download;
|
|
||||
|
|
||||
let (filename, uri) =
|
|
||||
if let Download { ref filename, ref uri, message: _ } = message {
|
|
||||
(filename, uri)
|
|
||||
} else {
|
|
||||
return Err(DownloadError::new(
|
|
||||
message.clone(),
|
|
||||
Some(anyhow!("Called with invalid variant")) ));
|
|
||||
};
|
|
||||
|
|
||||
// - get all informations to eventually existing file
|
|
||||
let mut from = file_size(filename).await;
|
|
||||
|
|
||||
// - get infos to uri
|
|
||||
let content_length = content_length(&mut client, uri).await
|
|
||||
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
|
|
||||
|
|
||||
if let Some(content_length) = content_length {
|
|
||||
if from != 0 && content_length - 1 <= from {
|
|
||||
return Ok(None);
|
|
||||
}
|
|
||||
} else {
|
|
||||
from = 0;
|
|
||||
}
|
|
||||
|
|
||||
// - do the neccessry request.
|
|
||||
let range_value: HeaderValue = format!("bytes={}-", from)
|
|
||||
. parse()
|
|
||||
. expect("Error creating range header value");
|
|
||||
let mut headers = HeaderMap::new();
|
|
||||
headers.insert(RANGE, range_value);
|
|
||||
|
|
||||
let mut response = request(&mut client, "GET", uri, headers).await
|
|
||||
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
|
|
||||
|
|
||||
// - open or create file
|
|
||||
// - download Data
|
|
||||
store_body( &mut open_outfile(&response.status(), filename).await
|
|
||||
, response.body_mut()
|
|
||||
, io_timeout )
|
|
||||
. await
|
|
||||
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
|
|
||||
|
|
||||
Ok(Some(message))
|
|
||||
}
|
|
||||
|
|
||||
pub(super) async fn body_bytes( mut client: HttpClient
|
|
||||
, mut message: ClientActorMessageHandle ) -> JoinSetResult {
|
|
||||
use ClientActorMessageHandle::GetData;
|
|
||||
|
|
||||
let uri =
|
|
||||
if let GetData { ref uri, buffer: _, message: _ } = message {
|
|
||||
uri
|
|
||||
} else {
|
|
||||
return Err(DownloadError::new(
|
|
||||
message.clone(),
|
|
||||
Some(anyhow!("Called with invalid variant")) ));
|
|
||||
};
|
|
||||
|
|
||||
let mut response = request( &mut client
|
|
||||
, "GET"
|
|
||||
, uri
|
|
||||
, HeaderMap::new() )
|
|
||||
. await
|
|
||||
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
|
|
||||
|
|
||||
// read body into Vec<u8>
|
|
||||
let body: Vec<u8> = BodyReader::new(response.body_mut())
|
|
||||
. bytes()
|
|
||||
. await
|
|
||||
. map_err(|e| DownloadError::new( message.clone()
|
|
||||
, Some(anyhow!(e.to_string())) ))?
|
|
||||
. to_vec();
|
|
||||
|
|
||||
let buffer =
|
|
||||
if let GetData { uri: _, ref mut buffer, message: _ } = message {
|
|
||||
buffer
|
|
||||
} else {
|
|
||||
return Err(DownloadError::new(
|
|
||||
message.clone(),
|
|
||||
Some(anyhow!("Called with invalid variant")) ));
|
|
||||
};
|
|
||||
|
|
||||
*buffer = Some(body);
|
|
||||
|
|
||||
Ok(Some(message))
|
|
||||
}
|
|
||||
|
|
||||
|
|
||||
|
|
||||
impl ClientActor {
|
|
||||
pub(super) fn new( concurrency_limit: usize |
|
||||
, timeout: Duration
|
|
||||
, receiver: mpsc::Receiver<ClientActorMessage>
|
|
||||
, abort_rx: oneshot::Receiver<JoinSetResult> ) -> anyhow::Result<Self> {
|
|
||||
let client = ServiceBuilder::new()
|
|
||||
// Add some layers.
|
|
||||
. concurrency_limit(concurrency_limit)
|
|
||||
. timeout(timeout)
|
|
||||
. layer(DecompressionLayer::new())
|
|
||||
// Make client compatible with the `tower-http` layers.
|
|
||||
. layer(HttpClientLayer)
|
|
||||
. service( reqwest::Client::builder()
|
|
||||
. redirect(Policy::limited(5))
|
|
||||
. build()? )
|
|
||||
. map_err(anyhow::Error::msg)
|
|
||||
. boxed_clone();
|
|
||||
|
|
||||
debug!("-> client: {:?}", client);
|
|
||||
|
|
||||
let mut tasks = JoinSet::new();
|
|
||||
|
|
||||
tasks.spawn(async move {
|
|
||||
let _ = abort_rx.await;
|
|
||||
Ok(None)
|
|
||||
});
|
|
||||
|
|
||||
let actions = HashMap::new();
|
|
||||
let actions_idx = 0;
|
|
||||
|
|
||||
Ok(Self {timeout, client, tasks, receiver, actions, actions_idx})
|
|
||||
}
|
|
||||
|
|
||||
async fn handle_message(&mut self, message: ClientActorMessage) {
|
|
||||
self.actions.insert(self.actions_idx, message);
|
|
||||
|
|
||||
use ClientActorMessage::{Download, GetData};
|
|
||||
|
|
||||
match self.actions.get(&self.actions_idx) {
|
|
||||
Some(Download { ref filename, ref uri, respond_to: _ }) => {
|
|
||||
// spawn a task that does the work
|
|
||||
let client = self.client.clone();
|
|
||||
let timeout = self.timeout;
|
|
||||
|
|
||||
let handle = ClientActorMessageHandle::Download {
|
|
||||
filename: filename.to_path_buf(),
|
|
||||
uri: uri.clone(),
|
|
||||
message: self.actions_idx,
|
|
||||
};
|
|
||||
|
|
||||
self.tasks.spawn(async move {
|
|
||||
download(client, handle, timeout).await
|
|
||||
});
|
|
||||
|
|
||||
self.actions_idx += 1;
|
|
||||
},
|
|
||||
|
|
||||
Some(GetData { ref uri, respond_to: _ }) => {
|
|
||||
// spawn a task that does the work
|
|
||||
let client = self.client.clone();
|
|
||||
|
|
||||
let handle = ClientActorMessageHandle::GetData {
|
|
||||
uri: uri.clone(),
|
|
||||
buffer: None,
|
|
||||
message: self.actions_idx,
|
|
||||
};
|
|
||||
|
|
||||
self.tasks.spawn(async move {
|
|
||||
body_bytes(client, handle).await
|
|
||||
});
|
|
||||
|
|
||||
self.actions_idx += 1;
|
|
||||
},
|
|
||||
|
|
||||
None => (),
|
|
||||
}
|
|
||||
}
|
|
||||
}
|
|
||||
|
|
||||
impl ClientActorHandle {
|
|
||||
pub(super) fn new(timeout: Duration) -> Self {
|
|
||||
let (sender, receiver) = mpsc::channel(1);
|
|
||||
let (abort, abort_rx) = oneshot::channel::<JoinSetResult>();
|
|
||||
let actor = ClientActor::new( 20
|
|
||||
, timeout
|
|
||||
, receiver
|
|
||||
, abort_rx )
|
|
||||
. expect("Client create error");
|
|
||||
tokio::spawn(run_client(actor));
|
|
||||
|
|
||||
Self { sender, abort }
|
|
||||
}
|
|
||||
|
|
||||
pub(super) fn stop(self) {
|
|
||||
let _ = self.abort.send(Ok(None));
|
|
||||
drop(self.sender);
|
|
||||
}
|
|
||||
|
|
||||
pub(super) async fn download(&self, filename: impl AsRef<Path>, uri: &Uri) {
|
|
||||
let filename = filename.as_ref().to_path_buf();
|
|
||||
let uri = uri.to_owned();
|
|
||||
let (send, receive) = oneshot::channel();
|
|
||||
let msg = ClientActorMessage::Download { filename, uri, respond_to: send };
|
|
||||
|
|
||||
let _ = self.sender.send(msg).await;
|
|
||||
receive.await.expect("Actor cancelled unexpected");
|
|
||||
}
|
|
||||
|
|
||||
pub(super) async fn body_bytes(&self, uri: &Uri) -> Option<Vec<u8>> {
|
|
||||
let uri = uri.to_owned();
|
|
||||
let (send, receive) = oneshot::channel();
|
|
||||
let msg = ClientActorMessage::GetData { uri, respond_to: send };
|
|
||||
|
|
||||
let _ = self.sender.send(msg).await;
|
|
||||
receive.await.expect("Actor cancelled unexpected")
|
|
||||
}
|
|
||||
}
|
|
||||
@ -1,37 +0,0 @@ |
|||||
use std::{error, fmt};
|
|
||||
|
|
||||
use crate::new_client::ClientActorMessageHandle;
|
|
||||
|
|
||||
|
|
||||
#[derive(Debug)]
|
|
||||
pub(super) struct DownloadError {
|
|
||||
pub(super) action: ClientActorMessageHandle,
|
|
||||
pub(super) source: Option<anyhow::Error>,
|
|
||||
}
|
|
||||
|
|
||||
|
|
||||
impl DownloadError {
|
|
||||
pub(super) fn new( action: ClientActorMessageHandle
|
|
||||
, source: Option<anyhow::Error> ) -> Self {
|
|
||||
let action = action.to_owned();
|
|
||||
Self { action, source }
|
|
||||
}
|
|
||||
}
|
|
||||
|
|
||||
impl error::Error for DownloadError {
|
|
||||
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
|
|
||||
match &self.source {
|
|
||||
None => None,
|
|
||||
Some(e) => Some(e.as_ref()),
|
|
||||
}
|
|
||||
}
|
|
||||
}
|
|
||||
|
|
||||
impl fmt::Display for DownloadError {
|
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||
match &self.source {
|
|
||||
None => write!(f, "download error: {:?}", self.action),
|
|
||||
Some(err) => write!(f, "download error ({:?}): {}", self.action, err),
|
|
||||
}
|
|
||||
}
|
|
||||
}
|
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue