Browse Source

huge progress with new code

main
Georg Hopp 11 months ago
parent
commit
c04d55d0d7
Signed by: ghopp GPG Key ID: 4C5D226768784538
  1. 2
      Cargo.lock
  2. 2
      Cargo.toml
  3. 675
      src/client.rs
  4. 16
      src/download_error.rs
  5. 193
      src/m3u8_download.rs
  6. 57
      src/main.rs
  7. 462
      src/new_client.rs
  8. 37
      src/new_download_error.rs
  9. 2
      src/process.rs

2
Cargo.lock

@ -509,7 +509,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hlsclient"
version = "0.2.0"
version = "0.2.1"
dependencies = [
"anyhow",
"clap",

2
Cargo.toml

@ -1,6 +1,6 @@
[package]
name = "hlsclient"
version = "0.2.0"
version = "0.2.1"
edition = "2021"
[dependencies]

675
src/client.rs

@ -1,11 +1,11 @@
use std::{io::ErrorKind, path::Path, time::Duration};
use std::{collections::HashMap, io::ErrorKind, path::{Path, PathBuf}, time::Duration};
use anyhow::anyhow;
use futures_util::StreamExt as _;
use http::{
header::{CONTENT_LENGTH, CONTENT_TYPE, RANGE},
request::Builder as RequestBuilder,
uri::{Authority, Scheme},
HeaderMap,
HeaderValue,
Request,
Response,
@ -13,296 +13,499 @@ use http::{
Uri
};
use http_body_util::BodyDataStream;
use m3u8_rs::{MediaPlaylist, MediaSegment, Playlist};
use reqwest::{redirect::Policy, Body};
use tokio::{
fs::{symlink_metadata, File},
io::AsyncWriteExt as _,
select,
sync::{mpsc, oneshot},
task::JoinSet,
time::timeout
};
use tower::{util::BoxCloneService, ServiceBuilder, ServiceExt as _};
use tower_http::decompression::{DecompressionBody, DecompressionLayer};
use tower_http_client::{client::BodyReader, ServiceExt as _};
use tower_reqwest::HttpClientLayer;
use crate::download_error::DownloadError;
use log::{log, Level};
use log::{debug, error, info};
type HttpClient = BoxCloneService<Request<Body>, Response<Body>, anyhow::Error>;
#[derive(Debug)]
pub(super) enum ClientActorMessage {
Download {
filename: PathBuf,
uri: Uri,
respond_to: oneshot::Sender<DownloadResult>,
},
GetData {
uri: Uri,
respond_to: oneshot::Sender<Option<Vec<u8>>>,
},
}
#[derive(Clone, Debug)]
pub(super) enum ClientActorMessageHandle {
Download {
filename: PathBuf,
uri: Uri,
state: Option<DownloadState>,
message: ActionIndex,
},
GetData {
uri: Uri,
buffer: Option<Vec<u8>>,
message: ActionIndex,
},
}
#[derive(Clone, Debug)]
pub struct State {
scheme: Scheme,
auth: Authority,
base_path: String,
pub enum DownloadState {
#[allow(dead_code)]
GotHead { content_type: Option<String> },
#[allow(dead_code)]
Responded { content_type: Option<String> },
#[allow(dead_code)]
Partial { content_type: Option<String> },
Done { content_type: Option<String> },
}
pub(super) type ActionIndex = u64;
type DownloadResult = Result<Option<DownloadState>, DownloadError>;
type JoinSetResult = Result<Option<ClientActorMessageHandle>, DownloadError>;
type HttpClient = BoxCloneService<Request<Body>, Response<DecompressionBody<Body>>, anyhow::Error>;
#[derive(Debug)]
struct ClientActor {
timeout: Duration,
client: HttpClient,
tasks: JoinSet<JoinSetResult>,
actions: HashMap<ActionIndex, ClientActorMessage>,
actions_idx: ActionIndex,
receiver: mpsc::Receiver<ClientActorMessage>,
}
impl State {
pub fn new(uri: &Uri, concurrency_limit: usize, timeout: Duration) -> anyhow::Result<Self>
{
let scheme = uri.scheme()
. ok_or(anyhow!("Problem scheme in m3u8 uri"))?;
let authority = uri.authority()
. ok_or(anyhow!("Problem authority in m3u8 uri"))?;
let base_path = Path::new(uri.path()).parent()
. ok_or(anyhow!("Path problem"))?
. to_str()
. ok_or(anyhow!("Path problem"))?;
let state = State {
scheme: scheme.clone(),
auth: authority.clone(),
base_path: base_path.to_string(),
timeout,
client: ServiceBuilder::new()
// Add some layers.
. concurrency_limit(concurrency_limit)
. timeout(timeout)
// Make client compatible with the `tower-http` layers.
. layer(HttpClientLayer)
. service(reqwest::Client::builder()
. redirect(Policy::limited(5))
. build()? )
. map_err(anyhow::Error::msg)
. boxed_clone(),
};
log!(Level::Debug, "-> state: {:?}", state);
Ok(state)
pub(super) struct ClientActorHandle {
sender: mpsc::Sender<ClientActorMessage>,
abort: oneshot::Sender<JoinSetResult>,
}
async fn run_client(mut actor: ClientActor) {
loop {
select! {
Some(join) = actor.tasks.join_next() => {
match join {
Err(e) => {
error!("FATAL Join failed: {}", e);
break
},
Ok(Err(e)) => {
info!("Retry failed download: {:?}", e);
// retry ... instead of responing here we could also respond
// with something that in turn would be used to retry...
let client = actor.client.clone();
actor.tasks.spawn(async move {
download(client, e.action, actor.timeout).await
});
},
// when the task finishes
Ok(Ok(Some(action))) => {
use ClientActorMessageHandle::{Download, GetData};
match action {
Download { filename: _, ref uri, state, ref message } => {
info!("Done download: {:?}", uri);
if let Some((_, message)) = actor.actions.remove_entry(message) {
use ClientActorMessage::Download;
match message {
Download { filename: _, uri: _, respond_to } => {
let _ = respond_to.send(Ok(state));
},
_ => panic!("Wrong variant ... this should never happen"),
}
} else {
panic!("Lost a message");
}
},
GetData { ref uri, buffer, ref message } => {
info!("Done download: {:?}", uri);
if let Some((_, message)) = actor.actions.remove_entry(message) {
use ClientActorMessage::GetData;
match message {
GetData { uri: _, respond_to } => {
let _ = respond_to.send(buffer);
},
_ => panic!("Wrong variant ... this should never happen"),
}
} else {
panic!("Lost a message");
}
},
}
},
// Got a stop message...here we still continue procession until the
// JoinSet is empty.
Ok(Ok(None)) => (),
};
}
Some(message) = actor.receiver.recv() => {
actor.handle_message(message).await;
}
else => {}
}
}
}
async fn file_size(filename: &Path) -> u64 {
// - get all informations to eventually existing file
let metadata = match symlink_metadata(filename).await {
Ok(metadata) => Some(metadata),
Err(error) => match error.kind() {
// If we can't write to a file we need to ... well theres nothing we can do
ErrorKind::PermissionDenied => panic!("Permission denied on: {:?}", filename),
_ => None,
}
};
pub(super) async fn get_m3u8_segment_uris(&mut self, path_and_query: &str)
-> anyhow::Result<Vec<Uri>>
{
let uri = Uri::builder()
. scheme(self.scheme.clone())
. authority(self.auth.clone())
. path_and_query(path_and_query)
. build()?;
metadata.map_or(0, |m| m.len())
}
async fn request( client: &mut HttpClient
, method: &str
, uri: &Uri
, headers: HeaderMap ) -> anyhow::Result<Response<DecompressionBody<Body>>> {
let mut request = RequestBuilder::new()
. method(method)
. uri(uri)
. body(Body::default())?;
let mut response = self.request(&uri, 0).await?;
request.headers_mut().extend(headers);
// read body into Vec<u8>
let body: Vec<u8> = BodyReader::new(response.body_mut())
. bytes().await?.to_vec();
debug!("Request: {:?}", request);
match m3u8_rs::parse_playlist(&body) {
Result::Err(e) => Err(anyhow!("m3u8 parse error: {}", e)),
let response = client.execute(request).await?;
Result::Ok((_, Playlist::MasterPlaylist(_))) =>
Err(anyhow!("Master playlist not supported now")),
debug!("Response: {:?}", response.headers());
Result::Ok((_, Playlist::MediaPlaylist(pl))) => {
Self::write_playlist(&uri, &pl).await?;
pl.segments.iter().map(|s| self.download_uri(s) ).collect()
},
}
anyhow::ensure!( response.status().is_success()
, "resonse status failed: {}"
, response.status() );
Ok(response)
}
async fn content_info( client: &mut HttpClient
, uri: &Uri ) -> anyhow::Result<(Option<u64>, Option<String>)> {
let head = request(client, "HEAD", uri, HeaderMap::new()).await?;
let content_length = head
. headers()
. get(CONTENT_LENGTH)
. map(|v| v . to_str()
. expect("unable to get CONTENT-LENGTH value")
. parse::<u64>()
. expect("unable to parse CONTENT-LENGTH value"));
let content_type = head
. headers()
. get(CONTENT_TYPE)
. map(|v| v . to_str()
. expect("unable to get CONTENT-LENGTH value")
. to_string());
Ok((content_length, content_type))
}
async fn open_outfile(status: &StatusCode, filename: &Path) -> File {
match status {
&StatusCode::PARTIAL_CONTENT =>
// Here we assume that this response only comes if the requested
// range was fullfillable and thus is the data range in the
// response. Thats why I do not check the content-range header.
// If that assumption does not hold this needs to be fixec.
File::options() . create(true)
. append(true)
. open(filename)
. await
. expect("can not create file for writing"),
_ =>
File::create(filename) . await
. expect("can not create file for writing"),
}
}
pub(super) async fn get_m3u8_segment(&mut self, uri: &Uri)
-> Result<Uri, DownloadError>
{
// I consider a missing path as fatal... there is absolutely nothing we can do about it
// and we need all files from the playlist.
let filename = Path::new(uri.path())
. file_name()
. expect("no filename in path_and_query");
let metadata = match symlink_metadata(filename).await {
Ok(metadata) => Some(metadata),
Err(error) => match error.kind() {
ErrorKind::PermissionDenied => panic!("Permission denied on: {:?}", filename),
_ => None,
async fn store_body( file: &mut File
, body: &mut DecompressionBody<Body>
, io_timeout: Duration ) -> anyhow::Result<()> {
let mut body = BodyDataStream::new(body);
loop {
// give timeout somehow... probably from client.
let data = timeout(io_timeout, body.next()).await?;
match data {
None => break,
Some(Err(e)) => {
return Err(anyhow!(e.to_string()));
}
};
let mut response = self.request( uri
, metadata.map_or(0, |m| m.len()) )
. await
. map_err(|e| DownloadError::new(uri.clone(), Some(e)))?;
match response.headers().get("x-finished") {
None => (),
Some(v) => if let Ok(v) = v.to_str() {
if v == "true" {
return Ok(uri.clone())
}
Some(Ok(data)) => {
file . write_all(data.as_ref()).await?;
file . flush().await?;
},
}
};
// We always need the content-type to be able to decide
let content_type = response.headers().get(CONTENT_TYPE)
. expect("No content-type header found in response")
. to_str()
. expect("Can't create &str from content-type header") ;
if content_type != "video/MP2T" {
let message = format!("unexpected content-type: {}", content_type);
log!(Level::Debug, "{}", message);
return Err(DownloadError::new( uri.clone()
, Some(anyhow!(message)) ));
}
Ok(())
}
let mut file = match response.status() {
StatusCode::PARTIAL_CONTENT =>
// Here we assume that this response only comes if the requested
// range was fullfillable and thus is the data range in the
// response. Thats why I do not check the content-range header.
// If that assumption does not hold this needs to be fixec.
File::options()
. create(true)
. append(true)
. open(filename)
. await
. expect("can not create file for writing"),
_ =>
File::create(filename).await
. expect("can not create file for writing"),
async fn download( mut client: HttpClient
, mut message: ClientActorMessageHandle
, io_timeout: Duration ) -> JoinSetResult {
use ClientActorMessageHandle::Download;
let (filename, uri) =
if let Download { ref filename, ref uri, state: _, message: _ } = message {
(filename, uri)
} else {
Err(DownloadError::new( message.clone()
, Some(anyhow!("Called with invalid variant")) ))?
};
// read body into file as stream
let mut body_stream = BodyDataStream::new(response.body_mut());
'label: loop {
let data = timeout(self.timeout, body_stream.next()).await
. map_err(|e| DownloadError::new(uri.clone(), Some(e.into())))?;
match data {
None => break 'label,
Some(Err(e)) =>
return Err(DownloadError::new(uri.clone(), Some(e.into()))),
Some(Ok(data)) => {
file.write_all(data.as_ref()).await
. map_err(|e|
DownloadError::new(uri.clone(), Some(e.into())) )?;
file.flush().await
. map_err(|e|
DownloadError::new(uri.clone(), Some(e.into())) )?;
},
}
};
// - get all informations to eventually existing file
let mut from = file_size(filename).await;
Ok(uri.clone())
}
// - get infos to uri
let (content_length, content_type) =
content_info(&mut client, uri).await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
fn download_uri(&self, segment: &MediaSegment) -> anyhow::Result<Uri>
{
match Uri::try_from(segment.uri.clone()) {
Ok(uri) => {
let scheme = uri.scheme().unwrap_or(&self.scheme);
let auth = uri.authority().unwrap_or(&self.auth);
let path_and_query = uri.path_and_query()
. ok_or(anyhow!("No path in Uri"))?;
Ok(Uri::builder() . scheme(scheme.clone())
. authority(auth.clone())
. path_and_query(path_and_query.clone())
. build()?)
}
if let Download { filename: _, uri: _, ref mut state, message: _ } = message {
let content_type = content_type.clone();
*state = Some(DownloadState::GotHead { content_type });
} else {
Err(DownloadError::new( message.clone()
, Some(anyhow!("Called with invalid variant")) ))?;
}
Err(_) => {
Ok(Uri::builder() . scheme(self.scheme.clone())
. authority(self.auth.clone())
. path_and_query(self.base_path.clone() + "/" + &segment.uri)
. build()?)
}
if let Some(content_length) = content_length {
if from != 0 && content_length - 1 <= from {
return Ok(None);
}
} else {
from = 0;
}
fn uri_relative_path(uri: &Uri) -> anyhow::Result<String>
{
Self::relative_path(uri.path())
// - do the neccessry request.
let range_value: HeaderValue = format!("bytes={}-", from)
. parse()
. expect("Error creating range header value");
let mut headers = HeaderMap::new();
headers.insert(RANGE, range_value);
let mut response = request(&mut client, "GET", uri, headers).await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
if let Download { filename: _, uri: _, ref mut state, message: _ } = message {
let content_type = content_type.clone();
*state = Some(DownloadState::Responded { content_type });
} else {
Err(DownloadError::new( message.clone()
, Some(anyhow!("Called with invalid variant")) ))?;
}
fn relative_path(path: &str) -> anyhow::Result<String>
{
let filename = Path::new(path)
. file_name()
. ok_or(anyhow!("name error"))?
. to_str()
. ok_or(anyhow!("Error getting filename from uri"))?;
Ok(filename.to_string())
// - open or create file
// - download Data
store_body( &mut open_outfile(&response.status(), filename).await
, response.body_mut()
, io_timeout )
. await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
if let Download { filename: _, uri: _, ref mut state, message: _ } = message {
let content_type = content_type.clone();
*state = Some(DownloadState::Done { content_type });
} else {
Err(DownloadError::new( message.clone()
, Some(anyhow!("Called with invalid variant")) ))?;
}
async fn request(&mut self, uri: &Uri, from: u64) -> anyhow::Result<Response<Body>>
{
let request = RequestBuilder::new()
. method("HEAD")
. uri(uri)
. body(Body::default())?;
log!(Level::Debug, "{:?}", request);
let response = self.client.execute(request).await?;
anyhow::ensure!( response.status().is_success()
, "resonse status failed: {}"
, response.status() );
log!(Level::Debug, "{:?}", response);
let content_length: u64 = response.headers().get(CONTENT_LENGTH)
. or(Some(&HeaderValue::from(0)))
. expect("No CONTENT-LENGTH in response")
. to_str()
. expect("unable to get CONTENT-LENGTH value")
. parse()
. expect("unable to parse CONTENT-LENGTH value");
Ok(Some(message))
}
if from != 0 && content_length - 1 <= from {
let response = Response::builder()
. header("x-finished", "true")
. body(Body::default())
. expect("Unable to create default response");
pub(super) async fn body_bytes( mut client: HttpClient
, mut message: ClientActorMessageHandle ) -> JoinSetResult {
use ClientActorMessageHandle::GetData;
let uri =
if let GetData { ref uri, buffer: _, message: _ } = message {
uri
} else {
return Err(DownloadError::new(
message.clone(),
Some(anyhow!("Called with invalid variant")) ));
};
let mut response = request( &mut client
, "GET"
, uri
, HeaderMap::new() )
. await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
// read body into Vec<u8>
let body: Vec<u8> = BodyReader::new(response.body_mut())
. bytes()
. await
. map_err(|e| DownloadError::new( message.clone()
, Some(anyhow!(e.to_string())) ))?
. to_vec();
let buffer =
if let GetData { uri: _, ref mut buffer, message: _ } = message {
buffer
} else {
return Err(DownloadError::new(
message.clone(),
Some(anyhow!("Called with invalid variant")) ));
};
*buffer = Some(body);
Ok(Some(message))
}
impl ClientActor {
pub(super) fn new( concurrency_limit: usize
, timeout: Duration
, receiver: mpsc::Receiver<ClientActorMessage>
, abort_rx: oneshot::Receiver<JoinSetResult> ) -> anyhow::Result<Self> {
let client = ServiceBuilder::new()
// Add some layers.
. concurrency_limit(concurrency_limit)
. timeout(timeout)
. layer(DecompressionLayer::new())
// Make client compatible with the `tower-http` layers.
. layer(HttpClientLayer)
. service( reqwest::Client::builder()
. redirect(Policy::limited(5))
. build()? )
. map_err(anyhow::Error::msg)
. boxed_clone();
debug!("-> client: {:?}", client);
let mut tasks = JoinSet::new();
tasks.spawn(async move {
let _ = abort_rx.await;
Ok(None)
});
let actions = HashMap::new();
let actions_idx = 0;
Ok(Self {timeout, client, tasks, receiver, actions, actions_idx})
}
async fn handle_message(&mut self, message: ClientActorMessage) {
self.actions.insert(self.actions_idx, message);
use ClientActorMessage::{Download, GetData};
log!(Level::Debug, "content_length: {}, from: {}", content_length, from);
match self.actions.get(&self.actions_idx) {
Some(Download { ref filename, ref uri, respond_to: _ }) => {
// spawn a task that does the work
let client = self.client.clone();
let timeout = self.timeout;
return Ok(response);
let handle = ClientActorMessageHandle::Download {
filename: filename.to_path_buf(),
uri: uri.clone(),
state: None,
message: self.actions_idx,
};
self.tasks.spawn(async move {
download(client, handle, timeout).await
});
self.actions_idx += 1;
},
Some(GetData { ref uri, respond_to: _ }) => {
// spawn a task that does the work
let client = self.client.clone();
let handle = ClientActorMessageHandle::GetData {
uri: uri.clone(),
buffer: None,
message: self.actions_idx,
};
self.tasks.spawn(async move {
body_bytes(client, handle).await
});
self.actions_idx += 1;
},
None => (),
}
}
}
let request = RequestBuilder::new()
. uri(uri)
. header(RANGE, format!("bytes={}-", from))
. body(Body::default())?;
log!(Level::Debug, "{:?}", request);
impl ClientActorHandle {
pub(super) fn new(concurrency: usize, timeout: Duration) -> Self {
let (sender, receiver) = mpsc::channel(1);
let (abort, abort_rx) = oneshot::channel::<JoinSetResult>();
let actor = ClientActor::new( concurrency
, timeout
, receiver
, abort_rx )
. expect("Client create error");
tokio::spawn(run_client(actor));
Self { sender, abort }
}
let response = self.client.execute(request).await?;
anyhow::ensure!( response.status().is_success()
, "resonse status failed: {}"
, response.status() );
log!(Level::Debug, "{:?}", response);
pub(super) fn stop(self) {
let _ = self.abort.send(Ok(None));
drop(self.sender);
}
Ok(response)
pub(super) async fn download( &self
, filename: impl AsRef<Path>
, uri: &Uri ) -> DownloadResult {
let filename = filename.as_ref().to_path_buf();
let uri = uri.to_owned();
let (send, receive) = oneshot::channel();
let msg = ClientActorMessage::Download { filename, uri, respond_to: send };
let _ = self.sender.send(msg).await;
receive.await.expect("Actor cancelled unexpected")
}
async fn write_playlist(uri: &Uri, playlist: &MediaPlaylist) -> anyhow::Result<()> {
let filename = Path::new(uri.path())
. file_name()
. ok_or(anyhow!("can't extract filename from uri"))?;
let mut file = File::create(filename).await?;
let segments: anyhow::Result<Vec<_>> = playlist.segments.iter().map(|s| {
let mut new_segment = s.clone();
Ok(match Uri::try_from(s.uri.clone()) {
Ok(uri) => {
new_segment.uri = Self::uri_relative_path(&uri)?;
new_segment
}
Err(_) => {
let uri = s.uri.split_once('?')
. map(|(s,_)| s)
. unwrap_or(&s.uri);
new_segment.uri = Self::relative_path(uri)?;
new_segment
}
})
}).collect();
let mut out_pl = playlist.clone();
out_pl.segments = segments?;
let mut file_data = vec![];
out_pl.write_to(&mut file_data)?;
file.write_all(&file_data).await?;
Ok(())
pub(super) async fn body_bytes(&self, uri: &Uri) -> Option<Vec<u8>> {
let uri = uri.to_owned();
let (send, receive) = oneshot::channel();
let msg = ClientActorMessage::GetData { uri, respond_to: send };
let _ = self.sender.send(msg).await;
receive.await.expect("Actor cancelled unexpected")
}
}

16
src/download_error.rs

@ -1,16 +1,20 @@
use std::{error, fmt};
use http::Uri;
use crate::client::ClientActorMessageHandle;
#[derive(Debug)]
pub(super) struct DownloadError {
pub(super) uri: Uri,
pub(super) action: ClientActorMessageHandle,
pub(super) source: Option<anyhow::Error>,
}
impl DownloadError {
pub(super) fn new(uri: Uri, source: Option<anyhow::Error>) -> Self {
Self { uri, source }
pub(super) fn new( action: ClientActorMessageHandle
, source: Option<anyhow::Error> ) -> Self {
let action = action.to_owned();
Self { action, source }
}
}
@ -26,8 +30,8 @@ impl error::Error for DownloadError {
impl fmt::Display for DownloadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.source {
None => write!(f, "download error: {}", self.uri),
Some(err) => write!(f, "download error ({}): {}", self.uri, err),
None => write!(f, "download error: {:?}", self.action),
Some(err) => write!(f, "download error ({:?}): {}", self.action, err),
}
}
}

193
src/m3u8_download.rs

@ -1,45 +1,204 @@
use std::{ffi::OsString, path::Path};
use std::path::{Path, PathBuf};
use anyhow::anyhow;
use futures_util::future::join_all;
use http::{uri::{Authority, Scheme}, Uri};
use log::debug;
use m3u8_rs::{MediaPlaylist, MediaSegment, Playlist};
use tokio::{io::AsyncWriteExt as _, fs::File};
use crate::client::{ClientActorHandle, DownloadState};
#[derive(Debug)]
pub(super) enum DownloadState {
Open, // Nothing is done.
Prepared, // The Uris to all .ts downloads are prepared.
#[derive(Clone, Debug)]
pub(super) enum TsState {
Created, // Nothing is done.
Failed, // The download has failed.
Ready, // All .ts downloads are done.
}
#[derive(Clone, Debug)]
struct TsPart {
filename: OsString,
url: Uri,
state: DownloadState,
#[allow(dead_code)]
filename: PathBuf,
#[allow(dead_code)]
uri: Uri,
#[allow(dead_code)]
state: TsState,
content_type: Option<String>,
}
struct M3u8Download {
#[derive(Debug)]
pub(super) struct M3u8Download {
#[allow(dead_code)]
index_uri: Uri,
#[allow(dead_code)]
scheme: Scheme,
#[allow(dead_code)]
auth: Authority,
#[allow(dead_code)]
base_path: String,
#[allow(dead_code)]
ts_parts: Vec<TsPart>,
}
impl TsPart {
fn new(uri: Uri) -> Self {
let filename = PathBuf::from(Path::new(uri.path())
. file_name()
. expect("no filename in path_and_query"));
let state = TsState::Created;
let content_type = None;
Self { filename, uri, state, content_type }
}
async fn download(&mut self, client: &ClientActorHandle) -> &Self {
let state = client.download(self.filename.clone(), &self.uri).await;
match state {
Ok(Some(DownloadState::Done { content_type })) => {
self.state = TsState::Ready;
self.content_type = content_type;
},
_ => self.state = TsState::Failed,
};
self
}
}
impl M3u8Download {
pub(super) fn new(uri: Uri) -> anyhow::Result<Self> {
let scheme = uri.scheme()
pub(super) async fn new(m3u8_data: Vec<u8>, index_uri: Uri) -> anyhow::Result<Self> {
let scheme = index_uri.scheme()
. ok_or(anyhow!("Problem scheme in m3u8 uri"))?
. clone();
let auth= uri.authority()
. to_owned();
let auth = index_uri.authority()
. ok_or(anyhow!("Problem authority in m3u8 uri"))?
. clone();
let base_path = Path::new(uri.path()).parent()
. to_owned();
let base_path = Path::new(index_uri.path()).parent()
. ok_or(anyhow!("Path problem"))?
. to_str()
. ok_or(anyhow!("Path problem"))?
. to_string();
let ts_parts = vec![];
Ok(Self {scheme, auth, base_path, ts_parts})
let mut ts_parts = vec![];
match m3u8_rs::parse_playlist(&m3u8_data) {
Result::Err(e) => Err(anyhow!("m3u8 parse error: {}", e))?,
Result::Ok((_, Playlist::MasterPlaylist(_))) =>
Err(anyhow!("Master playlist not supported now"))?,
Result::Ok((_, Playlist::MediaPlaylist(pl))) => {
Self::write_playlist(&index_uri, &pl).await?;
for segment in pl.segments.iter() {
let uri = Self::download_uri(&scheme, &auth, &base_path, segment)?;
ts_parts.push(TsPart::new(uri));
}
},
};
Ok(Self {index_uri, scheme, auth, base_path, ts_parts})
}
pub(super) fn index_uri(&self) -> &Uri {
&self.index_uri
}
pub(super) async fn download(&mut self, client: &ClientActorHandle) {
loop {
let unfinished: Vec<_> = self.ts_parts.iter_mut()
. filter_map(|p| match p.state {
TsState::Ready => if p.content_type != Some("video/MP2T".to_string()) {
Some(p.download(client))
} else {
None
}
_ => Some(p.download(client))
}).collect();
debug!("UNFINISHED NOW: {}", unfinished.len());
if unfinished.is_empty() { break; }
join_all(unfinished).await;
}
}
fn download_uri( scheme: &Scheme
, auth: &Authority
, base_path: &str
, segment: &MediaSegment ) -> anyhow::Result<Uri>
{
match Uri::try_from(segment.uri.clone()) {
Ok(uri) => {
let scheme = uri.scheme().unwrap_or(scheme);
let auth = uri.authority().unwrap_or(auth);
let path_and_query = uri.path_and_query()
. ok_or(anyhow!("No path in Uri"))?;
Ok(Uri::builder() . scheme(scheme.clone())
. authority(auth.clone())
. path_and_query(path_and_query.clone())
. build()?)
}
Err(_) => {
Ok(Uri::builder() . scheme(scheme.clone())
. authority(auth.clone())
. path_and_query(base_path.to_owned() + "/" + &segment.uri)
. build()?)
}
}
}
async fn write_playlist(uri: &Uri, playlist: &MediaPlaylist) -> anyhow::Result<()> {
let filename = Path::new(uri.path())
. file_name()
. ok_or(anyhow!("can't extract filename from uri"))?;
let mut file = File::create(filename).await?;
let segments: anyhow::Result<Vec<_>> = playlist.segments.iter().map(|s| {
let mut new_segment = s.clone();
Ok(match Uri::try_from(s.uri.clone()) {
Ok(uri) => {
new_segment.uri = Self::uri_relative_path(&uri)?;
new_segment
}
Err(_) => {
let uri = s.uri.split_once('?')
. map(|(s,_)| s)
. unwrap_or(&s.uri);
new_segment.uri = Self::relative_path(uri)?;
new_segment
}
})
}).collect();
let mut out_pl = playlist.clone();
out_pl.segments = segments?;
let mut file_data = vec![];
out_pl.write_to(&mut file_data)?;
file.write_all(&file_data).await?;
Ok(())
}
fn uri_relative_path(uri: &Uri) -> anyhow::Result<String>
{
Self::relative_path(uri.path())
}
fn relative_path(path: &str) -> anyhow::Result<String>
{
let filename = Path::new(path)
. file_name()
. ok_or(anyhow!("name error"))?
. to_str()
. ok_or(anyhow!("Error getting filename from uri"))?;
Ok(filename.to_string())
}
}

57
src/main.rs

@ -2,8 +2,6 @@ mod download_error;
mod client;
mod process;
mod m3u8_download;
mod new_client;
mod new_download_error;
use std::{
ffi::OsStr,
@ -15,10 +13,10 @@ use anyhow::anyhow;
use clap::Parser;
use env_logger::Env;
use http::Uri;
use new_client::ClientActorHandle;
use tokio::task::JoinSet;
use m3u8_download::M3u8Download;
use client::ClientActorHandle;
use log::{debug, error, info};
use log::{debug, info};
use process::{enter_download_dir, ffmpeg, remove_download_dir};
@ -72,8 +70,6 @@ async fn main() -> anyhow::Result<()> {
let timeout = Duration::from_secs(timeout);
let m3u8_uri = Uri::try_from(&args.url)?;
let m3u8_path_and_query = m3u8_uri.path_and_query()
. ok_or(anyhow!("Problem path and query in m3u8 uri"))?;
info!("Create and chdir into temporary download dir...");
@ -81,53 +77,22 @@ async fn main() -> anyhow::Result<()> {
info!("Creating an HTTP client with Tower layers...");
let mut state = client::State::new(&m3u8_uri, concurrency, timeout)?;
let client = ClientActorHandle::new(concurrency, timeout);
info!("Get segments...");
let mut segments = state.get_m3u8_segment_uris(m3u8_path_and_query.as_str()).await?;
let m3u8_data = client.body_bytes(&m3u8_uri).await
. ok_or(anyhow!("Unable to get body for: {}", m3u8_uri))?;
let mut download = M3u8Download::new(m3u8_data, m3u8_uri).await?;
debug!("M3u8Download: {:?}", download);
info!("Sending concurrent requests...");
let handle = ClientActorHandle::new(timeout);
handle.download("foo.m3u8", &m3u8_uri).await;
let body = handle.body_bytes(&m3u8_uri).await;
debug!("body: {:?}", body);
handle.stop();
let mut join_set = JoinSet::new();
while let Some(segment) = segments.pop() {
info!("Spawn task for: {}", segment);
let mut state = state.clone();
join_set.spawn(async move {
state.get_m3u8_segment(&segment).await
});
}
while let Some(result) = join_set.join_next().await {
match result {
Err(e) => {
error!("FATAL Join failed: {}", e);
break
},
Ok(Err(e)) => {
info!("Retry failed download: {:?}", e);
let mut state = state.clone();
join_set.spawn(async move {
state.get_m3u8_segment(&e.uri).await
});
},
Ok(Ok(v)) => {
info!("Done download: {}", v);
},
}
}
download.download(&client).await;
client.stop();
info!("Call ffmpeg to join ts files to single mp4...");
let status = ffmpeg(&name, &m3u8_uri).await?;
let status = ffmpeg(&name, download.index_uri()).await?;
debug!("ffmpeg status: {}", status);

462
src/new_client.rs

@ -1,462 +0,0 @@
use std::{collections::HashMap, io::ErrorKind, path::{Path, PathBuf}, time::Duration};
use anyhow::anyhow;
use futures_util::StreamExt as _;
use http::{
header::{CONTENT_LENGTH, RANGE},
request::Builder as RequestBuilder,
HeaderMap,
HeaderValue,
Request,
Response,
StatusCode,
Uri
};
use http_body_util::BodyDataStream;
use reqwest::{redirect::Policy, Body};
use tokio::{
fs::{symlink_metadata, File},
io::AsyncWriteExt as _,
select,
sync::{mpsc, oneshot},
task::JoinSet,
time::timeout
};
use tower::{util::BoxCloneService, ServiceBuilder, ServiceExt as _};
use tower_http::decompression::{DecompressionBody, DecompressionLayer};
use tower_http_client::{client::BodyReader, ServiceExt as _};
use tower_reqwest::HttpClientLayer;
use crate::{
new_download_error::DownloadError,
m3u8_download::DownloadState
};
use log::{debug, error, info};
#[derive(Debug)]
pub(super) enum ClientActorMessage {
Download {
filename: PathBuf,
uri: Uri,
respond_to: oneshot::Sender<DownloadState>,
},
GetData {
uri: Uri,
respond_to: oneshot::Sender<Option<Vec<u8>>>,
},
}
#[derive(Clone, Debug)]
pub(super) enum ClientActorMessageHandle {
Download {
filename: PathBuf,
uri: Uri,
message: ActionIndex,
},
GetData {
uri: Uri,
buffer: Option<Vec<u8>>,
message: ActionIndex,
},
}
pub(super) type ActionIndex = u64;
type JoinSetResult = Result<Option<ClientActorMessageHandle>, DownloadError>;
type HttpClient = BoxCloneService<Request<Body>, Response<DecompressionBody<Body>>, anyhow::Error>;
#[derive(Debug)]
struct ClientActor {
timeout: Duration,
client: HttpClient,
tasks: JoinSet<JoinSetResult>,
actions: HashMap<ActionIndex, ClientActorMessage>,
actions_idx: ActionIndex,
receiver: mpsc::Receiver<ClientActorMessage>,
}
pub(super) struct ClientActorHandle {
sender: mpsc::Sender<ClientActorMessage>,
abort: oneshot::Sender<JoinSetResult>,
}
async fn run_client(mut actor: ClientActor) {
loop {
select! {
Some(join) = actor.tasks.join_next() => {
match join {
Err(e) => {
error!("FATAL Join failed: {}", e);
break
},
Ok(Err(e)) => {
info!("Retry failed download: {:?}", e);
// retry ... instead of responing here we could also respond
// with something that in turn would be used to retry...
let client = actor.client.clone();
actor.tasks.spawn(async move {
download(client, e.action, actor.timeout).await
});
},
// when the task finishes
Ok(Ok(Some(action))) => {
info!("Done download: {:?}", action);
use ClientActorMessageHandle::{Download, GetData};
match action {
Download { filename: _, uri: _, ref message } => {
if let Some((_, message)) = actor.actions.remove_entry(message) {
use ClientActorMessage::Download;
match message {
Download { filename: _, uri: _, respond_to } => {
let _ = respond_to.send(DownloadState::Ready);
},
_ => panic!("Wrong variant ... this should never happen"),
}
} else {
panic!("Lost a message");
}
},
GetData { uri: _, buffer, ref message } => {
if let Some((_, message)) = actor.actions.remove_entry(message) {
use ClientActorMessage::GetData;
match message {
GetData { uri: _, respond_to } => {
let _ = respond_to.send(buffer);
},
_ => panic!("Wrong variant ... this should never happen"),
}
} else {
panic!("Lost a message");
}
},
}
},
// Got a stop message...here we still continue procession until the
// JoinSet is empty.
Ok(Ok(None)) => (),
};
}
Some(message) = actor.receiver.recv() => {
actor.handle_message(message).await;
}
else => {}
}
}
}
async fn file_size(filename: &Path) -> u64 {
// - get all informations to eventually existing file
let metadata = match symlink_metadata(filename).await {
Ok(metadata) => Some(metadata),
Err(error) => match error.kind() {
// If we can't write to a file we need to ... well theres nothing we can do
ErrorKind::PermissionDenied => panic!("Permission denied on: {:?}", filename),
_ => None,
}
};
metadata.map_or(0, |m| m.len())
}
async fn request( client: &mut HttpClient
, method: &str
, uri: &Uri
, headers: HeaderMap ) -> anyhow::Result<Response<DecompressionBody<Body>>> {
let mut request = RequestBuilder::new()
. method(method)
. uri(uri)
. body(Body::default())?;
request.headers_mut().extend(headers);
debug!("Request: {:?}", request);
let response = client.execute(request).await?;
debug!("Response: {:?}", response.headers());
anyhow::ensure!( response.status().is_success()
, "resonse status failed: {}"
, response.status() );
Ok(response)
}
async fn content_length( client: &mut HttpClient
, uri: &Uri ) -> anyhow::Result<Option<u64>> {
let head = request(client, "HEAD", uri, HeaderMap::new()).await?;
Ok(head . headers().get(CONTENT_LENGTH)
. map(|v| v . to_str()
. expect("unable to get CONTENT-LENGTH value")
. parse::<u64>()
. expect("unable to parse CONTENT-LENGTH value")))
}
async fn open_outfile(status: &StatusCode, filename: &Path) -> File {
match status {
&StatusCode::PARTIAL_CONTENT =>
// Here we assume that this response only comes if the requested
// range was fullfillable and thus is the data range in the
// response. Thats why I do not check the content-range header.
// If that assumption does not hold this needs to be fixec.
File::options() . create(true)
. append(true)
. open(filename)
. await
. expect("can not create file for writing"),
_ =>
File::create(filename) . await
. expect("can not create file for writing"),
}
}
async fn store_body( file: &mut File
, body: &mut DecompressionBody<Body>
, io_timeout: Duration ) -> anyhow::Result<()> {
let mut body = BodyDataStream::new(body);
loop {
// give timeout somehow... probably from client.
let data = timeout(io_timeout, body.next()).await?;
match data {
None => break,
Some(Err(e)) => {
return Err(anyhow!(e.to_string()));
}
Some(Ok(data)) => {
file . write_all(data.as_ref()).await?;
file . flush().await?;
},
}
};
Ok(())
}
async fn download( mut client: HttpClient
, message: ClientActorMessageHandle
, io_timeout: Duration ) -> JoinSetResult {
use ClientActorMessageHandle::Download;
let (filename, uri) =
if let Download { ref filename, ref uri, message: _ } = message {
(filename, uri)
} else {
return Err(DownloadError::new(
message.clone(),
Some(anyhow!("Called with invalid variant")) ));
};
// - get all informations to eventually existing file
let mut from = file_size(filename).await;
// - get infos to uri
let content_length = content_length(&mut client, uri).await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
if let Some(content_length) = content_length {
if from != 0 && content_length - 1 <= from {
return Ok(None);
}
} else {
from = 0;
}
// - do the neccessry request.
let range_value: HeaderValue = format!("bytes={}-", from)
. parse()
. expect("Error creating range header value");
let mut headers = HeaderMap::new();
headers.insert(RANGE, range_value);
let mut response = request(&mut client, "GET", uri, headers).await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
// - open or create file
// - download Data
store_body( &mut open_outfile(&response.status(), filename).await
, response.body_mut()
, io_timeout )
. await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
Ok(Some(message))
}
pub(super) async fn body_bytes( mut client: HttpClient
, mut message: ClientActorMessageHandle ) -> JoinSetResult {
use ClientActorMessageHandle::GetData;
let uri =
if let GetData { ref uri, buffer: _, message: _ } = message {
uri
} else {
return Err(DownloadError::new(
message.clone(),
Some(anyhow!("Called with invalid variant")) ));
};
let mut response = request( &mut client
, "GET"
, uri
, HeaderMap::new() )
. await
. map_err(|e| DownloadError::new(message.clone(), Some(e)))?;
// read body into Vec<u8>
let body: Vec<u8> = BodyReader::new(response.body_mut())
. bytes()
. await
. map_err(|e| DownloadError::new( message.clone()
, Some(anyhow!(e.to_string())) ))?
. to_vec();
let buffer =
if let GetData { uri: _, ref mut buffer, message: _ } = message {
buffer
} else {
return Err(DownloadError::new(
message.clone(),
Some(anyhow!("Called with invalid variant")) ));
};
*buffer = Some(body);
Ok(Some(message))
}
impl ClientActor {
pub(super) fn new( concurrency_limit: usize
, timeout: Duration
, receiver: mpsc::Receiver<ClientActorMessage>
, abort_rx: oneshot::Receiver<JoinSetResult> ) -> anyhow::Result<Self> {
let client = ServiceBuilder::new()
// Add some layers.
. concurrency_limit(concurrency_limit)
. timeout(timeout)
. layer(DecompressionLayer::new())
// Make client compatible with the `tower-http` layers.
. layer(HttpClientLayer)
. service( reqwest::Client::builder()
. redirect(Policy::limited(5))
. build()? )
. map_err(anyhow::Error::msg)
. boxed_clone();
debug!("-> client: {:?}", client);
let mut tasks = JoinSet::new();
tasks.spawn(async move {
let _ = abort_rx.await;
Ok(None)
});
let actions = HashMap::new();
let actions_idx = 0;
Ok(Self {timeout, client, tasks, receiver, actions, actions_idx})
}
async fn handle_message(&mut self, message: ClientActorMessage) {
self.actions.insert(self.actions_idx, message);
use ClientActorMessage::{Download, GetData};
match self.actions.get(&self.actions_idx) {
Some(Download { ref filename, ref uri, respond_to: _ }) => {
// spawn a task that does the work
let client = self.client.clone();
let timeout = self.timeout;
let handle = ClientActorMessageHandle::Download {
filename: filename.to_path_buf(),
uri: uri.clone(),
message: self.actions_idx,
};
self.tasks.spawn(async move {
download(client, handle, timeout).await
});
self.actions_idx += 1;
},
Some(GetData { ref uri, respond_to: _ }) => {
// spawn a task that does the work
let client = self.client.clone();
let handle = ClientActorMessageHandle::GetData {
uri: uri.clone(),
buffer: None,
message: self.actions_idx,
};
self.tasks.spawn(async move {
body_bytes(client, handle).await
});
self.actions_idx += 1;
},
None => (),
}
}
}
impl ClientActorHandle {
pub(super) fn new(timeout: Duration) -> Self {
let (sender, receiver) = mpsc::channel(1);
let (abort, abort_rx) = oneshot::channel::<JoinSetResult>();
let actor = ClientActor::new( 20
, timeout
, receiver
, abort_rx )
. expect("Client create error");
tokio::spawn(run_client(actor));
Self { sender, abort }
}
pub(super) fn stop(self) {
let _ = self.abort.send(Ok(None));
drop(self.sender);
}
pub(super) async fn download(&self, filename: impl AsRef<Path>, uri: &Uri) {
let filename = filename.as_ref().to_path_buf();
let uri = uri.to_owned();
let (send, receive) = oneshot::channel();
let msg = ClientActorMessage::Download { filename, uri, respond_to: send };
let _ = self.sender.send(msg).await;
receive.await.expect("Actor cancelled unexpected");
}
pub(super) async fn body_bytes(&self, uri: &Uri) -> Option<Vec<u8>> {
let uri = uri.to_owned();
let (send, receive) = oneshot::channel();
let msg = ClientActorMessage::GetData { uri, respond_to: send };
let _ = self.sender.send(msg).await;
receive.await.expect("Actor cancelled unexpected")
}
}

37
src/new_download_error.rs

@ -1,37 +0,0 @@
use std::{error, fmt};
use crate::new_client::ClientActorMessageHandle;
#[derive(Debug)]
pub(super) struct DownloadError {
pub(super) action: ClientActorMessageHandle,
pub(super) source: Option<anyhow::Error>,
}
impl DownloadError {
pub(super) fn new( action: ClientActorMessageHandle
, source: Option<anyhow::Error> ) -> Self {
let action = action.to_owned();
Self { action, source }
}
}
impl error::Error for DownloadError {
fn source(&self) -> Option<&(dyn error::Error + 'static)> {
match &self.source {
None => None,
Some(e) => Some(e.as_ref()),
}
}
}
impl fmt::Display for DownloadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.source {
None => write!(f, "download error: {:?}", self.action),
Some(err) => write!(f, "download error ({:?}): {}", self.action, err),
}
}
}

2
src/process.rs

@ -55,7 +55,7 @@ pub(super) async fn ffmpeg(name: &dyn AsRef<Path>, uri: &Uri) -> anyhow::Result<
let ffmpeg = ffmpeg.as_os_str();
let index = Path::new(uri.path()).file_name()
. ok_or(anyhow!("unable to get index filename from url"))?;
. ok_or(anyhow!("unable to get index filename from url"))?;
let index = Path::new(index).canonicalize()?;
let index = index.as_os_str();

Loading…
Cancel
Save