[wip] single runtime handling
This commit is contained in:
@@ -100,7 +100,7 @@ impl ActionExecutor {
|
||||
/// Execute an action for the given execution, with cancellation support.
|
||||
///
|
||||
/// When the `cancel_token` is triggered, the running process receives
|
||||
/// SIGINT → SIGTERM → SIGKILL with escalating grace periods.
|
||||
/// SIGTERM → SIGKILL with a short grace period.
|
||||
pub async fn execute_with_cancel(
|
||||
&self,
|
||||
execution_id: i64,
|
||||
@@ -139,7 +139,7 @@ impl ActionExecutor {
|
||||
};
|
||||
|
||||
// Attach the cancellation token so the process executor can monitor it
|
||||
context.cancel_token = Some(cancel_token);
|
||||
context.cancel_token = Some(cancel_token.clone());
|
||||
|
||||
// Execute the action
|
||||
// Note: execute_action should rarely return Err - most failures should be
|
||||
@@ -181,7 +181,16 @@ impl ActionExecutor {
|
||||
execution_id, result.exit_code, result.error, is_success
|
||||
);
|
||||
|
||||
if is_success {
|
||||
let was_cancelled = cancel_token.is_cancelled()
|
||||
|| result
|
||||
.error
|
||||
.as_deref()
|
||||
.is_some_and(|e| e.contains("cancelled"));
|
||||
|
||||
if was_cancelled {
|
||||
self.handle_execution_cancelled(execution_id, &result)
|
||||
.await?;
|
||||
} else if is_success {
|
||||
self.handle_execution_success(execution_id, &result).await?;
|
||||
} else {
|
||||
self.handle_execution_failure(execution_id, Some(&result), None)
|
||||
@@ -913,6 +922,51 @@ impl ActionExecutor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_execution_cancelled(
|
||||
&self,
|
||||
execution_id: i64,
|
||||
result: &ExecutionResult,
|
||||
) -> Result<()> {
|
||||
let exec_dir = self.artifact_manager.get_execution_dir(execution_id);
|
||||
let mut result_data = serde_json::json!({
|
||||
"succeeded": false,
|
||||
"cancelled": true,
|
||||
"exit_code": result.exit_code,
|
||||
"duration_ms": result.duration_ms,
|
||||
"error": result.error.clone().unwrap_or_else(|| "Execution cancelled by user".to_string()),
|
||||
});
|
||||
|
||||
if !result.stdout.is_empty() {
|
||||
result_data["stdout"] = serde_json::json!(result.stdout);
|
||||
}
|
||||
|
||||
if !result.stderr.trim().is_empty() {
|
||||
let stderr_path = exec_dir.join("stderr.log");
|
||||
result_data["stderr_log"] = serde_json::json!(stderr_path.to_string_lossy());
|
||||
}
|
||||
|
||||
if result.stdout_truncated {
|
||||
result_data["stdout_truncated"] = serde_json::json!(true);
|
||||
result_data["stdout_bytes_truncated"] =
|
||||
serde_json::json!(result.stdout_bytes_truncated);
|
||||
}
|
||||
if result.stderr_truncated {
|
||||
result_data["stderr_truncated"] = serde_json::json!(true);
|
||||
result_data["stderr_bytes_truncated"] =
|
||||
serde_json::json!(result.stderr_bytes_truncated);
|
||||
}
|
||||
|
||||
let input = UpdateExecutionInput {
|
||||
status: Some(ExecutionStatus::Cancelled),
|
||||
result: Some(result_data),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
ExecutionRepository::update(&self.pool, execution_id, input).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Update execution status
|
||||
async fn update_execution_status(
|
||||
&self,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Local Runtime Module
|
||||
//!
|
||||
//! Provides local execution capabilities by combining Process and Shell runtimes.
|
||||
//! Provides local execution capabilities by combining Process and Native runtimes.
|
||||
//! This module serves as a facade for all local process-based execution.
|
||||
//!
|
||||
//! The `ProcessRuntime` is used for Python (and other interpreted languages),
|
||||
@@ -8,10 +8,11 @@
|
||||
|
||||
use super::native::NativeRuntime;
|
||||
use super::process::ProcessRuntime;
|
||||
use super::shell::ShellRuntime;
|
||||
use super::{ExecutionContext, ExecutionResult, Runtime, RuntimeError, RuntimeResult};
|
||||
use async_trait::async_trait;
|
||||
use attune_common::models::runtime::{InterpreterConfig, RuntimeExecutionConfig};
|
||||
use attune_common::models::runtime::{
|
||||
InlineExecutionConfig, InlineExecutionStrategy, InterpreterConfig, RuntimeExecutionConfig,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
use tracing::{debug, info};
|
||||
|
||||
@@ -19,7 +20,7 @@ use tracing::{debug, info};
|
||||
pub struct LocalRuntime {
|
||||
native: NativeRuntime,
|
||||
python: ProcessRuntime,
|
||||
shell: ShellRuntime,
|
||||
shell: ProcessRuntime,
|
||||
}
|
||||
|
||||
impl LocalRuntime {
|
||||
@@ -34,6 +35,23 @@ impl LocalRuntime {
|
||||
args: vec![],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
};
|
||||
|
||||
let shell_config = RuntimeExecutionConfig {
|
||||
interpreter: InterpreterConfig {
|
||||
binary: "/bin/bash".to_string(),
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig {
|
||||
strategy: InlineExecutionStrategy::TempFile,
|
||||
extension: Some(".sh".to_string()),
|
||||
inject_shell_helpers: true,
|
||||
},
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
@@ -47,7 +65,12 @@ impl LocalRuntime {
|
||||
PathBuf::from("/opt/attune/packs"),
|
||||
PathBuf::from("/opt/attune/runtime_envs"),
|
||||
),
|
||||
shell: ShellRuntime::new(),
|
||||
shell: ProcessRuntime::new(
|
||||
"shell".to_string(),
|
||||
shell_config,
|
||||
PathBuf::from("/opt/attune/packs"),
|
||||
PathBuf::from("/opt/attune/runtime_envs"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +78,7 @@ impl LocalRuntime {
|
||||
pub fn with_runtimes(
|
||||
native: NativeRuntime,
|
||||
python: ProcessRuntime,
|
||||
shell: ShellRuntime,
|
||||
shell: ProcessRuntime,
|
||||
) -> Self {
|
||||
Self {
|
||||
native,
|
||||
@@ -76,7 +99,10 @@ impl LocalRuntime {
|
||||
);
|
||||
Ok(&self.python)
|
||||
} else if self.shell.can_execute(context) {
|
||||
debug!("Selected Shell runtime for action: {}", context.action_ref);
|
||||
debug!(
|
||||
"Selected Shell (ProcessRuntime) for action: {}",
|
||||
context.action_ref
|
||||
);
|
||||
Ok(&self.shell)
|
||||
} else {
|
||||
Err(RuntimeError::RuntimeNotFound(format!(
|
||||
|
||||
@@ -159,9 +159,9 @@ pub struct ExecutionContext {
|
||||
/// Format for output parsing
|
||||
pub output_format: OutputFormat,
|
||||
|
||||
/// Optional cancellation token for graceful process termination.
|
||||
/// When triggered, the executor sends SIGINT → SIGTERM → SIGKILL
|
||||
/// with escalating grace periods.
|
||||
/// Optional cancellation token for process termination.
|
||||
/// When triggered, the executor sends SIGTERM → SIGKILL
|
||||
/// with a short grace period.
|
||||
pub cancel_token: Option<CancellationToken>,
|
||||
}
|
||||
|
||||
|
||||
@@ -19,12 +19,18 @@ use super::{
|
||||
process_executor, ExecutionContext, ExecutionResult, Runtime, RuntimeError, RuntimeResult,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use attune_common::models::runtime::{EnvironmentConfig, RuntimeExecutionConfig};
|
||||
use attune_common::models::runtime::{
|
||||
EnvironmentConfig, InlineExecutionStrategy, RuntimeExecutionConfig,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::process::Command;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
fn bash_single_quote_escape(s: &str) -> String {
|
||||
s.replace('\'', "'\\''")
|
||||
}
|
||||
|
||||
/// A generic runtime driven by `RuntimeExecutionConfig` from the database.
|
||||
///
|
||||
/// Each `ProcessRuntime` instance corresponds to a row in the `runtime` table.
|
||||
@@ -437,6 +443,90 @@ impl ProcessRuntime {
|
||||
pub fn config(&self) -> &RuntimeExecutionConfig {
|
||||
&self.config
|
||||
}
|
||||
|
||||
fn build_shell_inline_wrapper(
|
||||
&self,
|
||||
merged_parameters: &HashMap<String, serde_json::Value>,
|
||||
code: &str,
|
||||
) -> RuntimeResult<String> {
|
||||
let mut script = String::new();
|
||||
script.push_str("#!/bin/bash\n");
|
||||
script.push_str("set -e\n\n");
|
||||
|
||||
script.push_str("# Action parameters\n");
|
||||
for (key, value) in merged_parameters {
|
||||
let value_str = match value {
|
||||
serde_json::Value::String(s) => s.clone(),
|
||||
serde_json::Value::Number(n) => n.to_string(),
|
||||
serde_json::Value::Bool(b) => b.to_string(),
|
||||
_ => serde_json::to_string(value)?,
|
||||
};
|
||||
let escaped = bash_single_quote_escape(&value_str);
|
||||
script.push_str(&format!(
|
||||
"export PARAM_{}='{}'\n",
|
||||
key.to_uppercase(),
|
||||
escaped
|
||||
));
|
||||
script.push_str(&format!("export {}='{}'\n", key, escaped));
|
||||
}
|
||||
script.push('\n');
|
||||
script.push_str("# Action code\n");
|
||||
script.push_str(code);
|
||||
|
||||
Ok(script)
|
||||
}
|
||||
|
||||
async fn materialize_inline_code(
|
||||
&self,
|
||||
execution_id: i64,
|
||||
merged_parameters: &HashMap<String, serde_json::Value>,
|
||||
code: &str,
|
||||
effective_config: &RuntimeExecutionConfig,
|
||||
) -> RuntimeResult<(PathBuf, bool)> {
|
||||
let inline_dir = std::env::temp_dir().join("attune").join("inline_actions");
|
||||
tokio::fs::create_dir_all(&inline_dir).await.map_err(|e| {
|
||||
RuntimeError::ExecutionFailed(format!(
|
||||
"Failed to create inline action directory {}: {}",
|
||||
inline_dir.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let extension = effective_config
|
||||
.inline_execution
|
||||
.extension
|
||||
.as_deref()
|
||||
.unwrap_or("");
|
||||
let extension = if extension.is_empty() {
|
||||
String::new()
|
||||
} else if extension.starts_with('.') {
|
||||
extension.to_string()
|
||||
} else {
|
||||
format!(".{}", extension)
|
||||
};
|
||||
|
||||
let inline_path = inline_dir.join(format!("exec_{}{}", execution_id, extension));
|
||||
let inline_code = if effective_config.inline_execution.inject_shell_helpers {
|
||||
self.build_shell_inline_wrapper(merged_parameters, code)?
|
||||
} else {
|
||||
code.to_string()
|
||||
};
|
||||
|
||||
tokio::fs::write(&inline_path, inline_code)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
RuntimeError::ExecutionFailed(format!(
|
||||
"Failed to write inline action file {}: {}",
|
||||
inline_path.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok((
|
||||
inline_path,
|
||||
effective_config.inline_execution.inject_shell_helpers,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -661,7 +751,7 @@ impl Runtime for ProcessRuntime {
|
||||
};
|
||||
let prepared_params =
|
||||
parameter_passing::prepare_parameters(&merged_parameters, &mut env, param_config)?;
|
||||
let parameters_stdin = prepared_params.stdin_content();
|
||||
let mut parameters_stdin = prepared_params.stdin_content();
|
||||
|
||||
// Determine working directory: use context override, or pack dir
|
||||
let working_dir = context
|
||||
@@ -677,6 +767,7 @@ impl Runtime for ProcessRuntime {
|
||||
});
|
||||
|
||||
// Build the command based on whether we have a file or inline code
|
||||
let mut temp_inline_file: Option<PathBuf> = None;
|
||||
let cmd = if let Some(ref code_path) = context.code_path {
|
||||
// File-based execution: interpreter [args] <action_file>
|
||||
debug!("Executing file: {}", code_path.display());
|
||||
@@ -688,13 +779,38 @@ impl Runtime for ProcessRuntime {
|
||||
&env,
|
||||
)
|
||||
} else if let Some(ref code) = context.code {
|
||||
// Inline code execution: interpreter -c <code>
|
||||
debug!("Executing inline code ({} bytes)", code.len());
|
||||
let mut cmd = process_executor::build_inline_command(&interpreter, code, &env);
|
||||
if let Some(dir) = working_dir {
|
||||
cmd.current_dir(dir);
|
||||
match effective_config.inline_execution.strategy {
|
||||
InlineExecutionStrategy::Direct => {
|
||||
debug!("Executing inline code directly ({} bytes)", code.len());
|
||||
let mut cmd = process_executor::build_inline_command(&interpreter, code, &env);
|
||||
if let Some(dir) = working_dir {
|
||||
cmd.current_dir(dir);
|
||||
}
|
||||
cmd
|
||||
}
|
||||
InlineExecutionStrategy::TempFile => {
|
||||
debug!("Executing inline code via temp file ({} bytes)", code.len());
|
||||
let (inline_path, consumes_parameters) = self
|
||||
.materialize_inline_code(
|
||||
context.execution_id,
|
||||
&merged_parameters,
|
||||
code,
|
||||
effective_config,
|
||||
)
|
||||
.await?;
|
||||
if consumes_parameters {
|
||||
parameters_stdin = None;
|
||||
}
|
||||
temp_inline_file = Some(inline_path.clone());
|
||||
process_executor::build_action_command(
|
||||
&interpreter,
|
||||
&effective_config.interpreter.args,
|
||||
&inline_path,
|
||||
working_dir,
|
||||
&env,
|
||||
)
|
||||
}
|
||||
}
|
||||
cmd
|
||||
} else {
|
||||
// No code_path and no inline code — try treating entry_point as a file
|
||||
// relative to the pack's actions directory
|
||||
@@ -737,7 +853,7 @@ impl Runtime for ProcessRuntime {
|
||||
|
||||
// Execute with streaming output capture (with optional cancellation support).
|
||||
// Secrets are already merged into parameters — no separate secrets arg needed.
|
||||
process_executor::execute_streaming_cancellable(
|
||||
let result = process_executor::execute_streaming_cancellable(
|
||||
cmd,
|
||||
&HashMap::new(),
|
||||
parameters_stdin,
|
||||
@@ -747,7 +863,13 @@ impl Runtime for ProcessRuntime {
|
||||
context.output_format,
|
||||
context.cancel_token.clone(),
|
||||
)
|
||||
.await
|
||||
.await;
|
||||
|
||||
if let Some(path) = temp_inline_file {
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn setup(&self) -> RuntimeResult<()> {
|
||||
@@ -836,7 +958,8 @@ impl Runtime for ProcessRuntime {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use attune_common::models::runtime::{
|
||||
DependencyConfig, EnvironmentConfig, InterpreterConfig, RuntimeExecutionConfig,
|
||||
DependencyConfig, EnvironmentConfig, InlineExecutionConfig, InlineExecutionStrategy,
|
||||
InterpreterConfig, RuntimeExecutionConfig,
|
||||
};
|
||||
use attune_common::models::{OutputFormat, ParameterDelivery, ParameterFormat};
|
||||
use std::collections::HashMap;
|
||||
@@ -849,6 +972,11 @@ mod tests {
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig {
|
||||
strategy: InlineExecutionStrategy::TempFile,
|
||||
extension: Some(".sh".to_string()),
|
||||
inject_shell_helpers: true,
|
||||
},
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: HashMap::new(),
|
||||
@@ -862,6 +990,7 @@ mod tests {
|
||||
args: vec!["-u".to_string()],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: Some(EnvironmentConfig {
|
||||
env_type: "virtualenv".to_string(),
|
||||
dir_name: ".venv".to_string(),
|
||||
@@ -1104,6 +1233,7 @@ mod tests {
|
||||
args: vec![],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: HashMap::new(),
|
||||
@@ -1183,6 +1313,53 @@ mod tests {
|
||||
assert!(result.stdout.contains("inline shell code"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_inline_code_with_merged_inputs() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
let runtime = ProcessRuntime::new(
|
||||
"shell".to_string(),
|
||||
make_shell_config(),
|
||||
temp_dir.path().to_path_buf(),
|
||||
temp_dir.path().join("runtime_envs"),
|
||||
);
|
||||
|
||||
let context = ExecutionContext {
|
||||
execution_id: 30,
|
||||
action_ref: "adhoc.test_inputs".to_string(),
|
||||
parameters: {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("name".to_string(), serde_json::json!("Alice"));
|
||||
map
|
||||
},
|
||||
env: HashMap::new(),
|
||||
secrets: {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("api_key".to_string(), serde_json::json!("secret-123"));
|
||||
map
|
||||
},
|
||||
timeout: Some(10),
|
||||
working_dir: None,
|
||||
entry_point: "inline".to_string(),
|
||||
code: Some("echo \"$name/$api_key/$PARAM_NAME/$PARAM_API_KEY\"".to_string()),
|
||||
code_path: None,
|
||||
runtime_name: Some("shell".to_string()),
|
||||
runtime_config_override: None,
|
||||
runtime_env_dir_suffix: None,
|
||||
selected_runtime_version: None,
|
||||
max_stdout_bytes: 1024 * 1024,
|
||||
max_stderr_bytes: 1024 * 1024,
|
||||
parameter_delivery: ParameterDelivery::default(),
|
||||
parameter_format: ParameterFormat::default(),
|
||||
output_format: OutputFormat::default(),
|
||||
cancel_token: None,
|
||||
};
|
||||
|
||||
let result = runtime.execute(context).await.unwrap();
|
||||
assert_eq!(result.exit_code, 0);
|
||||
assert!(result.stdout.contains("Alice/secret-123/Alice/secret-123"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_entry_point_fallback() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -9,9 +9,8 @@
|
||||
//!
|
||||
//! When a `CancellationToken` is provided, the executor monitors it alongside
|
||||
//! the running process. On cancellation:
|
||||
//! 1. SIGINT is sent to the process (allows graceful shutdown)
|
||||
//! 2. After a 10-second grace period, SIGTERM is sent if the process hasn't exited
|
||||
//! 3. After another 5-second grace period, SIGKILL is sent as a last resort
|
||||
//! 1. SIGTERM is sent to the process immediately
|
||||
//! 2. After a 5-second grace period, SIGKILL is sent as a last resort
|
||||
|
||||
use super::{BoundedLogWriter, ExecutionResult, OutputFormat, RuntimeResult};
|
||||
use std::collections::HashMap;
|
||||
@@ -71,7 +70,7 @@ pub async fn execute_streaming(
|
||||
/// - Writing parameters (with secrets merged in) to stdin
|
||||
/// - Streaming stdout/stderr with bounded log collection
|
||||
/// - Timeout management
|
||||
/// - Graceful cancellation via SIGINT → SIGTERM → SIGKILL escalation
|
||||
/// - Prompt cancellation via SIGTERM → SIGKILL escalation
|
||||
/// - Output format parsing (JSON, YAML, JSONL, text)
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -199,35 +198,23 @@ pub async fn execute_streaming_cancellable(
|
||||
tokio::select! {
|
||||
result = timed_wait => (result, false),
|
||||
_ = token.cancelled() => {
|
||||
// Cancellation requested — escalate signals to the child process.
|
||||
info!("Cancel signal received, sending SIGINT to process");
|
||||
// Cancellation requested — terminate the child process promptly.
|
||||
info!("Cancel signal received, sending SIGTERM to process");
|
||||
if let Some(pid) = child_pid {
|
||||
send_signal(pid, libc::SIGINT);
|
||||
send_signal(pid, libc::SIGTERM);
|
||||
}
|
||||
|
||||
// Grace period: wait up to 10s for the process to exit after SIGINT.
|
||||
match timeout(std::time::Duration::from_secs(10), child.wait()).await {
|
||||
// Grace period: wait up to 5s for the process to exit after SIGTERM.
|
||||
match timeout(std::time::Duration::from_secs(5), child.wait()).await {
|
||||
Ok(status) => (Ok(status), true),
|
||||
Err(_) => {
|
||||
// Still alive — escalate to SIGTERM
|
||||
warn!("Process did not exit after SIGINT + 10s grace period, sending SIGTERM");
|
||||
// Last resort — SIGKILL
|
||||
warn!("Process did not exit after SIGTERM + 5s, sending SIGKILL");
|
||||
if let Some(pid) = child_pid {
|
||||
send_signal(pid, libc::SIGTERM);
|
||||
}
|
||||
|
||||
// Final grace period: wait up to 5s for SIGTERM
|
||||
match timeout(std::time::Duration::from_secs(5), child.wait()).await {
|
||||
Ok(status) => (Ok(status), true),
|
||||
Err(_) => {
|
||||
// Last resort — SIGKILL
|
||||
warn!("Process did not exit after SIGTERM + 5s, sending SIGKILL");
|
||||
if let Some(pid) = child_pid {
|
||||
send_signal(pid, libc::SIGKILL);
|
||||
}
|
||||
// Wait indefinitely for the SIGKILL to take effect
|
||||
(Ok(child.wait().await), true)
|
||||
}
|
||||
send_signal(pid, libc::SIGKILL);
|
||||
}
|
||||
// Wait indefinitely for the SIGKILL to take effect
|
||||
(Ok(child.wait().await), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use attune_common::runtime_detection::runtime_in_filter;
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::PgPool;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -44,7 +44,6 @@ use crate::registration::WorkerRegistration;
|
||||
use crate::runtime::local::LocalRuntime;
|
||||
use crate::runtime::native::NativeRuntime;
|
||||
use crate::runtime::process::ProcessRuntime;
|
||||
use crate::runtime::shell::ShellRuntime;
|
||||
use crate::runtime::RuntimeRegistry;
|
||||
use crate::secrets::SecretManager;
|
||||
use crate::version_verify;
|
||||
@@ -89,8 +88,11 @@ pub struct WorkerService {
|
||||
in_flight_tasks: Arc<Mutex<JoinSet<()>>>,
|
||||
/// Maps execution ID → CancellationToken for running processes.
|
||||
/// When a cancel request arrives, the token is triggered, causing
|
||||
/// the process executor to send SIGINT → SIGTERM → SIGKILL.
|
||||
/// the process executor to send SIGTERM → SIGKILL.
|
||||
cancel_tokens: Arc<Mutex<HashMap<i64, CancellationToken>>>,
|
||||
/// Tracks cancellation requests that arrived before the in-memory token
|
||||
/// for an execution had been registered.
|
||||
pending_cancellations: Arc<Mutex<HashSet<i64>>>,
|
||||
}
|
||||
|
||||
impl WorkerService {
|
||||
@@ -263,9 +265,29 @@ impl WorkerService {
|
||||
if runtime_registry.list_runtimes().is_empty() {
|
||||
info!("No runtimes loaded from database, registering built-in defaults");
|
||||
|
||||
// Shell runtime (always available)
|
||||
runtime_registry.register(Box::new(ShellRuntime::new()));
|
||||
info!("Registered built-in Shell runtime");
|
||||
// Shell runtime (always available) via generic ProcessRuntime
|
||||
let shell_runtime = ProcessRuntime::new(
|
||||
"shell".to_string(),
|
||||
attune_common::models::runtime::RuntimeExecutionConfig {
|
||||
interpreter: attune_common::models::runtime::InterpreterConfig {
|
||||
binary: "/bin/bash".to_string(),
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: attune_common::models::runtime::InlineExecutionConfig {
|
||||
strategy: attune_common::models::runtime::InlineExecutionStrategy::TempFile,
|
||||
extension: Some(".sh".to_string()),
|
||||
inject_shell_helpers: true,
|
||||
},
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
},
|
||||
packs_base_dir.clone(),
|
||||
runtime_envs_dir.clone(),
|
||||
);
|
||||
runtime_registry.register(Box::new(shell_runtime));
|
||||
info!("Registered built-in shell ProcessRuntime");
|
||||
|
||||
// Native runtime (for compiled binaries)
|
||||
runtime_registry.register(Box::new(NativeRuntime::new()));
|
||||
@@ -379,6 +401,7 @@ impl WorkerService {
|
||||
execution_semaphore: Arc::new(Semaphore::new(max_concurrent_tasks)),
|
||||
in_flight_tasks: Arc::new(Mutex::new(JoinSet::new())),
|
||||
cancel_tokens: Arc::new(Mutex::new(HashMap::new())),
|
||||
pending_cancellations: Arc::new(Mutex::new(HashSet::new())),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -755,6 +778,7 @@ impl WorkerService {
|
||||
let semaphore = self.execution_semaphore.clone();
|
||||
let in_flight = self.in_flight_tasks.clone();
|
||||
let cancel_tokens = self.cancel_tokens.clone();
|
||||
let pending_cancellations = self.pending_cancellations.clone();
|
||||
|
||||
// Spawn the consumer loop as a background task so start() can return
|
||||
let handle = tokio::spawn(async move {
|
||||
@@ -768,6 +792,7 @@ impl WorkerService {
|
||||
let semaphore = semaphore.clone();
|
||||
let in_flight = in_flight.clone();
|
||||
let cancel_tokens = cancel_tokens.clone();
|
||||
let pending_cancellations = pending_cancellations.clone();
|
||||
|
||||
async move {
|
||||
let execution_id = envelope.payload.execution_id;
|
||||
@@ -794,6 +819,16 @@ impl WorkerService {
|
||||
let mut tokens = cancel_tokens.lock().await;
|
||||
tokens.insert(execution_id, cancel_token.clone());
|
||||
}
|
||||
{
|
||||
let pending = pending_cancellations.lock().await;
|
||||
if pending.contains(&execution_id) {
|
||||
info!(
|
||||
"Execution {} already had a pending cancel request; cancelling immediately",
|
||||
execution_id
|
||||
);
|
||||
cancel_token.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
// Spawn the actual execution as a background task so this
|
||||
// handler returns immediately, acking the message and freeing
|
||||
@@ -819,6 +854,8 @@ impl WorkerService {
|
||||
// Remove the cancel token now that execution is done
|
||||
let mut tokens = cancel_tokens.lock().await;
|
||||
tokens.remove(&execution_id);
|
||||
let mut pending = pending_cancellations.lock().await;
|
||||
pending.remove(&execution_id);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
@@ -1060,6 +1097,7 @@ impl WorkerService {
|
||||
|
||||
let consumer_for_task = consumer.clone();
|
||||
let cancel_tokens = self.cancel_tokens.clone();
|
||||
let pending_cancellations = self.pending_cancellations.clone();
|
||||
let queue_name_for_log = queue_name.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
@@ -1071,11 +1109,17 @@ impl WorkerService {
|
||||
.consume_with_handler(
|
||||
move |envelope: MessageEnvelope<ExecutionCancelRequestedPayload>| {
|
||||
let cancel_tokens = cancel_tokens.clone();
|
||||
let pending_cancellations = pending_cancellations.clone();
|
||||
|
||||
async move {
|
||||
let execution_id = envelope.payload.execution_id;
|
||||
info!("Received cancel request for execution {}", execution_id);
|
||||
|
||||
{
|
||||
let mut pending = pending_cancellations.lock().await;
|
||||
pending.insert(execution_id);
|
||||
}
|
||||
|
||||
let tokens = cancel_tokens.lock().await;
|
||||
if let Some(token) = tokens.get(&execution_id) {
|
||||
info!("Triggering cancellation for execution {}", execution_id);
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
//! This keeps the pack directory clean and read-only.
|
||||
|
||||
use attune_common::models::runtime::{
|
||||
DependencyConfig, EnvironmentConfig, InterpreterConfig, RuntimeExecutionConfig,
|
||||
DependencyConfig, EnvironmentConfig, InlineExecutionConfig, InterpreterConfig,
|
||||
RuntimeExecutionConfig,
|
||||
};
|
||||
use attune_worker::runtime::process::ProcessRuntime;
|
||||
use attune_worker::runtime::ExecutionContext;
|
||||
@@ -26,6 +27,7 @@ fn make_python_config() -> RuntimeExecutionConfig {
|
||||
args: vec!["-u".to_string()],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: Some(EnvironmentConfig {
|
||||
env_type: "virtualenv".to_string(),
|
||||
dir_name: ".venv".to_string(),
|
||||
@@ -59,6 +61,7 @@ fn make_shell_config() -> RuntimeExecutionConfig {
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
//! Tests that verify stdout/stderr are properly truncated when they exceed
|
||||
//! configured size limits, preventing OOM issues with large output.
|
||||
|
||||
use attune_common::models::runtime::{InterpreterConfig, RuntimeExecutionConfig};
|
||||
use attune_common::models::runtime::{
|
||||
InlineExecutionConfig, InterpreterConfig, RuntimeExecutionConfig,
|
||||
};
|
||||
use attune_worker::runtime::process::ProcessRuntime;
|
||||
use attune_worker::runtime::{ExecutionContext, Runtime, ShellRuntime};
|
||||
use std::collections::HashMap;
|
||||
@@ -17,6 +19,7 @@ fn make_python_process_runtime(packs_base_dir: PathBuf) -> ProcessRuntime {
|
||||
args: vec!["-u".to_string()],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
@@ -270,6 +273,7 @@ async fn test_shell_process_runtime_truncation() {
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
//! These tests verify that secrets are NOT exposed in process environment
|
||||
//! or command-line arguments, ensuring secure secret passing via stdin.
|
||||
|
||||
use attune_common::models::runtime::{InterpreterConfig, RuntimeExecutionConfig};
|
||||
use attune_common::models::runtime::{
|
||||
InlineExecutionConfig, InterpreterConfig, RuntimeExecutionConfig,
|
||||
};
|
||||
use attune_worker::runtime::process::ProcessRuntime;
|
||||
use attune_worker::runtime::shell::ShellRuntime;
|
||||
use attune_worker::runtime::{ExecutionContext, Runtime};
|
||||
@@ -18,6 +20,7 @@ fn make_python_process_runtime(packs_base_dir: PathBuf) -> ProcessRuntime {
|
||||
args: vec!["-u".to_string()],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
@@ -440,6 +443,7 @@ echo "PASS: No secrets in environment"
|
||||
args: vec![],
|
||||
file_extension: Some(".sh".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
@@ -520,6 +524,7 @@ print(json.dumps({"leaked": leaked}))
|
||||
args: vec!["-u".to_string()],
|
||||
file_extension: Some(".py".to_string()),
|
||||
},
|
||||
inline_execution: InlineExecutionConfig::default(),
|
||||
environment: None,
|
||||
dependencies: None,
|
||||
env_vars: std::collections::HashMap::new(),
|
||||
|
||||
Reference in New Issue
Block a user