Wire thread count and error behavior settings into executor
This commit is contained in:
@@ -33,6 +33,8 @@ pub struct BatchResult {
|
||||
pub struct PipelineExecutor {
|
||||
cancel_flag: Arc<AtomicBool>,
|
||||
pause_flag: Arc<AtomicBool>,
|
||||
thread_count: usize,
|
||||
pause_on_error: bool,
|
||||
}
|
||||
|
||||
impl PipelineExecutor {
|
||||
@@ -40,6 +42,8 @@ impl PipelineExecutor {
|
||||
Self {
|
||||
cancel_flag: Arc::new(AtomicBool::new(false)),
|
||||
pause_flag: Arc::new(AtomicBool::new(false)),
|
||||
thread_count: num_cpus(),
|
||||
pause_on_error: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,6 +51,8 @@ impl PipelineExecutor {
|
||||
Self {
|
||||
cancel_flag,
|
||||
pause_flag: Arc::new(AtomicBool::new(false)),
|
||||
thread_count: num_cpus(),
|
||||
pause_on_error: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,9 +63,19 @@ impl PipelineExecutor {
|
||||
Self {
|
||||
cancel_flag,
|
||||
pause_flag,
|
||||
thread_count: num_cpus(),
|
||||
pause_on_error: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_thread_count(&mut self, count: usize) {
|
||||
self.thread_count = if count == 0 { num_cpus() } else { count };
|
||||
}
|
||||
|
||||
pub fn set_pause_on_error(&mut self, pause: bool) {
|
||||
self.pause_on_error = pause;
|
||||
}
|
||||
|
||||
pub fn execute<F>(&self, job: &ProcessingJob, mut on_progress: F) -> Result<BatchResult>
|
||||
where
|
||||
F: FnMut(ProgressUpdate) + Send,
|
||||
@@ -122,6 +138,9 @@ impl PipelineExecutor {
|
||||
Err(e) => {
|
||||
result.failed += 1;
|
||||
result.errors.push((file_name, e.to_string()));
|
||||
if self.pause_on_error {
|
||||
self.pause_flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -198,3 +217,9 @@ impl Default for PipelineExecutor {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn num_cpus() -> usize {
|
||||
std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(1)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user