attune-system/attune
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

node running, runtime version awareness

Browse Source
This commit is contained in:
David Culbreth
2026-02-25 23:24:07 -06:00
parent e89b5991ec
commit 495b81236a
54 changed files with 4308 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/worker/Cargo.toml
View File

@@ -26,6 +26,7 @@ clap = { workspace = true }
lapin = { workspace = true }
reqwest = { workspace = true }
hostname = "0.4"
regex = { workspace = true }
async-trait = { workspace = true }
thiserror = { workspace = true }
aes-gcm = { workspace = true }

207
crates/worker/src/env_setup.rs
View File

@@ -8,6 +8,19 @@
//! The goal is to ensure environments are ready *before* the first execution,
//! eliminating the first-run penalty and potential permission errors that occur
//! when setup is deferred to execution time.
//!
//! ## Version-Aware Environments
//!
//! When runtime versions are registered (e.g., Python 3.11, 3.12, 3.13), this
//! module creates per-version environments at:
//! `{runtime_envs_dir}/{pack_ref}/{runtime_name}-{version}`
//!
//! For example: `/opt/attune/runtime_envs/my_pack/python-3.12`
//!
//! This ensures that different versions maintain isolated environments with
//! their own interpreter binaries and installed dependencies. A base (unversioned)
//! environment is also created for backward compatibility with actions that don't
//! declare a version constraint.
use std::collections::{HashMap, HashSet};
use std::path::Path;
@@ -15,11 +28,14 @@ use std::path::Path;
use sqlx::PgPool;
use tracing::{debug, error, info, warn};
use attune_common::models::RuntimeVersion;
use attune_common::mq::PackRegisteredPayload;
use attune_common::repositories::action::ActionRepository;
use attune_common::repositories::pack::PackRepository;
use attune_common::repositories::runtime::RuntimeRepository;
use attune_common::repositories::runtime_version::RuntimeVersionRepository;
use attune_common::repositories::{FindById, List};
use attune_common::runtime_detection::runtime_in_filter;
// Re-export the utility that the API also uses so callers can reach it from
// either crate without adding a direct common dependency for this one function.
@@ -96,6 +112,26 @@ pub async fn scan_and_setup_all_environments(
}
};
// Load all runtime versions, indexed by runtime ID
let version_map: HashMap<i64, Vec<RuntimeVersion>> =
match RuntimeVersionRepository::list(db_pool).await {
Ok(versions) => {
let mut map: HashMap<i64, Vec<RuntimeVersion>> = HashMap::new();
for v in versions {
map.entry(v.runtime).or_default().push(v);
}
map
}
Err(e) => {
warn!(
"Failed to load runtime versions from database: {}. \
Version-specific environments will not be created.",
e
);
HashMap::new()
}
};
info!("Found {} registered pack(s) to scan", packs.len());
for pack in &packs {
@@ -109,6 +145,7 @@ pub async fn scan_and_setup_all_environments(
packs_base_dir,
runtime_envs_dir,
&runtime_map,
&version_map,
)
.await;
@@ -164,13 +201,13 @@ pub async fn setup_environments_for_registered_pack(
return pack_result;
}
// Filter to runtimes this worker supports
// Filter to runtimes this worker supports (alias-aware matching)
let target_runtimes: Vec<&String> = event
.runtime_names
.iter()
.filter(|name| {
if let Some(filter) = runtime_filter {
filter.contains(name)
runtime_in_filter(name, filter)
} else {
true
}
@@ -219,6 +256,7 @@ pub async fn setup_environments_for_registered_pack(
continue;
}
// Set up base (unversioned) environment
let env_dir = runtime_envs_dir.join(&event.pack_ref).join(rt_name);
let process_runtime = ProcessRuntime::new(
@@ -248,6 +286,19 @@ pub async fn setup_environments_for_registered_pack(
pack_result.errors.push(msg);
}
}
// Set up per-version environments for available runtime versions
setup_version_environments(
db_pool,
rt.id,
rt_name,
&event.pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
}
pack_result
@@ -256,7 +307,8 @@ pub async fn setup_environments_for_registered_pack(
/// Internal helper: set up environments for a single pack during the startup scan.
///
/// Discovers which runtimes the pack's actions use, filters by this worker's
/// capabilities, and creates any missing environments.
/// capabilities, and creates any missing environments. Also creates per-version
/// environments for runtimes that have registered versions.
#[allow(clippy::too_many_arguments)]
async fn setup_environments_for_pack(
db_pool: &PgPool,
@@ -266,6 +318,7 @@ async fn setup_environments_for_pack(
packs_base_dir: &Path,
runtime_envs_dir: &Path,
runtime_map: &HashMap<i64, attune_common::models::Runtime>,
version_map: &HashMap<i64, Vec<RuntimeVersion>>,
) -> PackEnvSetupResult {
let mut pack_result = PackEnvSetupResult {
pack_ref: pack_ref.to_string(),
@@ -327,6 +380,25 @@ async fn setup_environments_for_pack(
&mut pack_result,
)
.await;
// Also set up version-specific environments
let versions = match RuntimeVersionRepository::find_available_by_runtime(
db_pool, runtime_id,
)
.await
{
Ok(v) => v,
Err(_) => Vec::new(),
};
setup_version_environments_from_list(
&versions,
&rt_name,
pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
continue;
}
Ok(None) => {
@@ -353,6 +425,22 @@ async fn setup_environments_for_pack(
&mut pack_result,
)
.await;
// Set up per-version environments for available versions of this runtime
if let Some(versions) = version_map.get(&runtime_id) {
let available_versions: Vec<RuntimeVersion> =
versions.iter().filter(|v| v.available).cloned().collect();
setup_version_environments_from_list(
&available_versions,
&rt_name,
pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
}
}
if !pack_result.environments_created.is_empty() {
@@ -377,9 +465,9 @@ async fn process_runtime_for_pack(
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
// Apply worker runtime filter
// Apply worker runtime filter (alias-aware matching)
if let Some(filter) = runtime_filter {
if !filter.iter().any(|f| f == rt_name) {
if !runtime_in_filter(rt_name, filter) {
debug!(
"Runtime '{}' not in worker filter, skipping for pack '{}'",
rt_name, pack_ref,
@@ -430,6 +518,115 @@ async fn process_runtime_for_pack(
}
}
/// Set up per-version environments for a runtime, given a list of available versions.
///
/// For each available version, creates an environment at:
/// `{runtime_envs_dir}/{pack_ref}/{runtime_name}-{version}`
///
/// This uses the version's own `execution_config` (which may specify a different
/// interpreter binary, environment create command, etc.).
#[allow(clippy::too_many_arguments)]
async fn setup_version_environments_from_list(
versions: &[RuntimeVersion],
rt_name: &str,
pack_ref: &str,
pack_dir: &Path,
packs_base_dir: &Path,
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
if versions.is_empty() {
return;
}
for version in versions {
let version_exec_config = version.parsed_execution_config();
// Skip versions with no environment config and no dependencies
if version_exec_config.environment.is_none()
&& !version_exec_config.has_dependencies(pack_dir)
{
debug!(
"Version '{}' {} has no environment config, skipping for pack '{}'",
version.runtime_ref, version.version, pack_ref,
);
continue;
}
let version_env_suffix = format!("{}-{}", rt_name, version.version);
let version_env_dir = runtime_envs_dir.join(pack_ref).join(&version_env_suffix);
let version_runtime = ProcessRuntime::new(
rt_name.to_string(),
version_exec_config,
packs_base_dir.to_path_buf(),
runtime_envs_dir.to_path_buf(),
);
match version_runtime
.setup_pack_environment(pack_dir, &version_env_dir)
.await
{
Ok(()) => {
info!(
"Version environment '{}' ready for pack '{}'",
version_env_suffix, pack_ref,
);
pack_result.environments_created.push(version_env_suffix);
}
Err(e) => {
let msg = format!(
"Failed to set up version environment '{}' for pack '{}': {}",
version_env_suffix, pack_ref, e,
);
warn!("{}", msg);
pack_result.errors.push(msg);
}
}
}
}
/// Set up per-version environments for a runtime by querying the database.
///
/// This is a convenience wrapper around `setup_version_environments_from_list`
/// that queries available versions from the database first. Used in the
/// pack.registered event handler where we don't have a pre-loaded version map.
#[allow(clippy::too_many_arguments)]
async fn setup_version_environments(
db_pool: &PgPool,
runtime_id: i64,
rt_name: &str,
pack_ref: &str,
pack_dir: &Path,
packs_base_dir: &Path,
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
let versions =
match RuntimeVersionRepository::find_available_by_runtime(db_pool, runtime_id).await {
Ok(v) => v,
Err(e) => {
debug!(
"Failed to load versions for runtime '{}' (id {}): {}. \
Skipping version-specific environments.",
rt_name, runtime_id, e,
);
return;
}
};
setup_version_environments_from_list(
&versions,
rt_name,
pack_ref,
pack_dir,
packs_base_dir,
runtime_envs_dir,
pack_result,
)
.await;
}
/// Determine the runtime filter from the `ATTUNE_WORKER_RUNTIMES` environment variable.
///
/// Returns `None` if the variable is not set (meaning all runtimes are accepted).

120
crates/worker/src/executor.rs
View File

@@ -2,11 +2,24 @@
//!
//! Coordinates the execution of actions by managing the runtime,
//! loading action data, preparing execution context, and collecting results.
//!
//! ## Runtime Version Selection
//!
//! When an action declares a `runtime_version_constraint` (e.g., `">=3.12"`),
//! the executor queries the `runtime_version` table for all versions of the
//! action's runtime and uses [`select_best_version`] to pick the highest
//! available version satisfying the constraint. The selected version's
//! `execution_config` is passed through the `ExecutionContext` as an override
//! so the `ProcessRuntime` uses version-specific interpreter binaries,
//! environment commands, etc.
use attune_common::error::{Error, Result};
use attune_common::models::runtime::RuntimeExecutionConfig;
use attune_common::models::{runtime::Runtime as RuntimeModel, Action, Execution, ExecutionStatus};
use attune_common::repositories::execution::{ExecutionRepository, UpdateExecutionInput};
use attune_common::repositories::runtime_version::RuntimeVersionRepository;
use attune_common::repositories::{FindById, Update};
use attune_common::version_matching::select_best_version;
use std::path::PathBuf as StdPathBuf;
use serde_json::Value as JsonValue;
@@ -365,6 +378,15 @@ impl ActionExecutor {
let runtime_name = runtime_record.as_ref().map(|r| r.name.to_lowercase());
// --- Runtime Version Resolution ---
// If the action declares a runtime_version_constraint (e.g., ">=3.12"),
// query all registered versions for this runtime and select the best
// match. The selected version's execution_config overrides the parent
// runtime's config so the ProcessRuntime uses a version-specific
// interpreter binary, environment commands, etc.
let (runtime_config_override, runtime_env_dir_suffix, selected_runtime_version) =
self.resolve_runtime_version(&runtime_record, action).await;
// Determine the pack directory for this action
let pack_dir = self.packs_base_dir.join(&action.pack_ref);
@@ -446,6 +468,9 @@ impl ActionExecutor {
code,
code_path,
runtime_name,
runtime_config_override,
runtime_env_dir_suffix,
selected_runtime_version,
max_stdout_bytes: self.max_stdout_bytes,
max_stderr_bytes: self.max_stderr_bytes,
parameter_delivery: action.parameter_delivery,
@@ -456,6 +481,101 @@ impl ActionExecutor {
Ok(context)
}
/// Resolve the best runtime version for an action, if applicable.
///
/// Returns a tuple of:
/// - Optional `RuntimeExecutionConfig` override (from the selected version)
/// - Optional env dir suffix (e.g., `"python-3.12"`) for per-version isolation
/// - Optional version string for logging (e.g., `"3.12"`)
///
/// If the action has no `runtime_version_constraint`, or no versions are
/// registered for its runtime, all three are `None` and the parent runtime's
/// config is used as-is.
async fn resolve_runtime_version(
&self,
runtime_record: &Option<RuntimeModel>,
action: &Action,
) -> (
Option<RuntimeExecutionConfig>,
Option<String>,
Option<String>,
) {
let runtime = match runtime_record {
Some(r) => r,
None => return (None, None, None),
};
// Query all versions for this runtime
let versions = match RuntimeVersionRepository::find_by_runtime(&self.pool, runtime.id).await
{
Ok(v) if !v.is_empty() => v,
Ok(_) => {
// No versions registered — use parent runtime config as-is
if action.runtime_version_constraint.is_some() {
warn!(
"Action '{}' declares runtime_version_constraint '{}' but runtime '{}' \
has no registered versions. Using parent runtime config.",
action.r#ref,
action.runtime_version_constraint.as_deref().unwrap_or(""),
runtime.name,
);
}
return (None, None, None);
}
Err(e) => {
warn!(
"Failed to load runtime versions for runtime '{}' (id {}): {}. \
Using parent runtime config.",
runtime.name, runtime.id, e,
);
return (None, None, None);
}
};
let constraint = action.runtime_version_constraint.as_deref();
match select_best_version(&versions, constraint) {
Some(selected) => {
let version_config = selected.parsed_execution_config();
let rt_name = runtime.name.to_lowercase();
let env_suffix = format!("{}-{}", rt_name, selected.version);
info!(
"Selected runtime version '{}' (id {}) for action '{}' \
(constraint: {}, runtime: '{}'). Env dir suffix: '{}'",
selected.version,
selected.id,
action.r#ref,
constraint.unwrap_or("none"),
runtime.name,
env_suffix,
);
(
Some(version_config),
Some(env_suffix),
Some(selected.version.clone()),
)
}
None => {
if let Some(c) = constraint {
warn!(
"No available runtime version matches constraint '{}' for action '{}' \
(runtime: '{}'). Using parent runtime config as fallback.",
c, action.r#ref, runtime.name,
);
} else {
debug!(
"No default or available version found for runtime '{}'. \
Using parent runtime config.",
runtime.name,
);
}
(None, None, None)
}
}
}
/// Execute the action using the runtime registry
async fn execute_action(&self, context: ExecutionContext) -> Result<ExecutionResult> {
debug!("Executing action: {}", context.action_ref);

1
crates/worker/src/lib.rs
View File

@@ -11,6 +11,7 @@ pub mod registration;
pub mod runtime;
pub mod secrets;
pub mod service;
pub mod version_verify;
// Re-export commonly used types
pub use executor::ActionExecutor;

7
crates/worker/src/runtime/local.rs
View File

@@ -36,6 +36,7 @@ impl LocalRuntime {
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
Self {
@@ -168,6 +169,9 @@ mod tests {
code: Some("#!/bin/bash\necho 'hello from shell'".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -197,6 +201,9 @@ mod tests {
code: Some("some code".to_string()),
code_path: None,
runtime_name: Some("unknown".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),

34
crates/worker/src/runtime/mod.rs
View File

@@ -9,6 +9,18 @@
//! as separate Rust types. Instead, the `ProcessRuntime` handles all
//! languages by using the interpreter, environment, and dependency
//! configuration stored in the database.
//!
//! ## Runtime Version Selection
//!
//! When an action declares a `runtime_version_constraint` (e.g., `">=3.12"`),
//! the executor resolves the best matching `RuntimeVersion` from the database
//! and passes its `execution_config` through `ExecutionContext::runtime_config_override`.
//! The `ProcessRuntime` uses this override instead of its built-in config,
//! enabling version-specific interpreter binaries, environment commands, etc.
//!
//! The environment directory is also overridden to include the version suffix
//! (e.g., `python-3.12` instead of `python`) so that different versions
//! maintain isolated environments.
pub mod dependency;
pub mod local;
@@ -26,6 +38,7 @@ pub use process::ProcessRuntime;
pub use shell::ShellRuntime;
use async_trait::async_trait;
use attune_common::models::runtime::RuntimeExecutionConfig;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
@@ -112,6 +125,24 @@ pub struct ExecutionContext {
/// Runtime name (python, shell, etc.) - used to select the correct runtime
pub runtime_name: Option<String>,
/// Optional override of the runtime's execution config, set when a specific
/// runtime version has been selected (e.g., Python 3.12 vs the parent
/// "Python" runtime). When present, `ProcessRuntime` uses this config
/// instead of its built-in one for interpreter resolution, environment
/// setup, and dependency management.
#[serde(skip)]
pub runtime_config_override: Option<RuntimeExecutionConfig>,
/// Optional override of the environment directory suffix. When a specific
/// runtime version is selected, the env dir includes the version
/// (e.g., `python-3.12` instead of `python`) for per-version isolation.
/// Format: just the directory name, not the full path.
pub runtime_env_dir_suffix: Option<String>,
/// The selected runtime version string for logging/diagnostics
/// (e.g., "3.12.1"). `None` means the parent runtime config is used as-is.
pub selected_runtime_version: Option<String>,
/// Maximum stdout size in bytes (for log truncation)
#[serde(default = "default_max_log_bytes")]
pub max_stdout_bytes: usize,
@@ -154,6 +185,9 @@ impl ExecutionContext {
code,
code_path: None,
runtime_name: None,
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),

129
crates/worker/src/runtime/process.rs
View File

@@ -94,6 +94,7 @@ impl ProcessRuntime {
}
/// Get the interpreter path, checking for an external pack environment first.
#[cfg(test)]
fn resolve_interpreter(&self, pack_dir: &Path, env_dir: Option<&Path>) -> PathBuf {
self.config.resolve_interpreter_with_env(pack_dir, env_dir)
}
@@ -472,24 +473,52 @@ impl Runtime for ProcessRuntime {
}
async fn execute(&self, context: ExecutionContext) -> RuntimeResult<ExecutionResult> {
info!(
"Executing action '{}' (execution_id: {}) with runtime '{}', \
parameter delivery: {:?}, format: {:?}, output format: {:?}",
context.action_ref,
context.execution_id,
self.runtime_name,
context.parameter_delivery,
context.parameter_format,
context.output_format,
);
// Determine the effective execution config: use the version-specific
// override if the executor resolved a specific runtime version for this
// action, otherwise fall back to this ProcessRuntime's built-in config.
let effective_config: &RuntimeExecutionConfig = context
.runtime_config_override
.as_ref()
.unwrap_or(&self.config);
if let Some(ref ver) = context.selected_runtime_version {
info!(
"Executing action '{}' (execution_id: {}) with runtime '{}' version {}, \
parameter delivery: {:?}, format: {:?}, output format: {:?}",
context.action_ref,
context.execution_id,
self.runtime_name,
ver,
context.parameter_delivery,
context.parameter_format,
context.output_format,
);
} else {
info!(
"Executing action '{}' (execution_id: {}) with runtime '{}', \
parameter delivery: {:?}, format: {:?}, output format: {:?}",
context.action_ref,
context.execution_id,
self.runtime_name,
context.parameter_delivery,
context.parameter_format,
context.output_format,
);
}
let pack_ref = self.extract_pack_ref(&context.action_ref);
let pack_dir = self.packs_base_dir.join(pack_ref);
// Compute external env_dir for this pack/runtime combination.
// Pattern: {runtime_envs_dir}/{pack_ref}/{runtime_name}
let env_dir = self.env_dir_for_pack(pack_ref);
let env_dir_opt = if self.config.environment.is_some() {
// When a specific runtime version is selected, the env dir includes a
// version suffix (e.g., "python-3.12") for per-version isolation.
// Pattern: {runtime_envs_dir}/{pack_ref}/{runtime_name[-version]}
let env_dir = if let Some(ref suffix) = context.runtime_env_dir_suffix {
self.runtime_envs_dir.join(pack_ref).join(suffix)
} else {
self.env_dir_for_pack(pack_ref)
};
let env_dir_opt = if effective_config.environment.is_some() {
Some(env_dir.as_path())
} else {
None
@@ -499,7 +528,7 @@ impl Runtime for ProcessRuntime {
// (scanning all registered packs) or via pack.registered MQ events when a
// new pack is installed. We only log a warning here if the expected
// environment directory is missing so operators can investigate.
if self.config.environment.is_some() && pack_dir.exists() && !env_dir.exists() {
if effective_config.environment.is_some() && pack_dir.exists() && !env_dir.exists() {
warn!(
"Runtime environment for pack '{}' not found at {}. \
The environment should have been created at startup or on pack registration. \
@@ -512,8 +541,8 @@ impl Runtime for ProcessRuntime {
// If the environment directory exists but contains a broken interpreter
// (e.g. broken symlinks from a venv created in a different container),
// attempt to recreate it before resolving the interpreter.
if self.config.environment.is_some() && env_dir.exists() && pack_dir.exists() {
if let Some(ref env_cfg) = self.config.environment {
if effective_config.environment.is_some() && env_dir.exists() && pack_dir.exists() {
if let Some(ref env_cfg) = effective_config.environment {
if let Some(ref interp_template) = env_cfg.interpreter_path {
let mut vars = std::collections::HashMap::new();
vars.insert("env_dir", env_dir.to_string_lossy().to_string());
@@ -550,8 +579,18 @@ impl Runtime for ProcessRuntime {
e,
);
} else {
// Recreate the environment
match self.setup_pack_environment(&pack_dir, &env_dir).await {
// Recreate the environment using a temporary ProcessRuntime
// with the effective (possibly version-specific) config.
let setup_runtime = ProcessRuntime::new(
self.runtime_name.clone(),
effective_config.clone(),
self.packs_base_dir.clone(),
self.runtime_envs_dir.clone(),
);
match setup_runtime
.setup_pack_environment(&pack_dir, &env_dir)
.await
{
Ok(()) => {
info!(
"Successfully recreated environment for pack '{}' at {}",
@@ -575,18 +614,37 @@ impl Runtime for ProcessRuntime {
}
}
let interpreter = self.resolve_interpreter(&pack_dir, env_dir_opt);
let interpreter = effective_config.resolve_interpreter_with_env(&pack_dir, env_dir_opt);
info!(
"Resolved interpreter: {} (env_dir: {}, env_exists: {}, pack_dir: {})",
"Resolved interpreter: {} (env_dir: {}, env_exists: {}, pack_dir: {}, version: {})",
interpreter.display(),
env_dir.display(),
env_dir.exists(),
pack_dir.display(),
context
.selected_runtime_version
.as_deref()
.unwrap_or("default"),
);
// Prepare environment and parameters according to delivery method
let mut env = context.env.clone();
// Inject runtime-specific environment variables from execution_config.
// These are template-based (e.g., NODE_PATH={env_dir}/node_modules) and
// resolved against the current pack/env directories.
if !effective_config.env_vars.is_empty() {
let vars = effective_config.build_template_vars_with_env(&pack_dir, env_dir_opt);
for (key, value_template) in &effective_config.env_vars {
let resolved = RuntimeExecutionConfig::resolve_template(value_template, &vars);
debug!(
"Setting runtime env var: {}={} (template: {})",
key, resolved, value_template
);
env.insert(key.clone(), resolved);
}
}
let param_config = ParameterDeliveryConfig {
delivery: context.parameter_delivery,
format: context.parameter_format,
@@ -614,7 +672,7 @@ impl Runtime for ProcessRuntime {
debug!("Executing file: {}", code_path.display());
process_executor::build_action_command(
&interpreter,
&self.config.interpreter.args,
&effective_config.interpreter.args,
code_path,
working_dir,
&env,
@@ -635,7 +693,7 @@ impl Runtime for ProcessRuntime {
debug!("Executing action file: {}", action_file.display());
process_executor::build_action_command(
&interpreter,
&self.config.interpreter.args,
&effective_config.interpreter.args,
&action_file,
working_dir,
&env,
@@ -781,6 +839,7 @@ mod tests {
},
environment: None,
dependencies: None,
env_vars: HashMap::new(),
}
}
@@ -813,6 +872,7 @@ mod tests {
"{manifest_path}".to_string(),
],
}),
env_vars: HashMap::new(),
}
}
@@ -837,6 +897,9 @@ mod tests {
code: None,
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024,
max_stderr_bytes: 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -868,6 +931,9 @@ mod tests {
code: None,
code_path: Some(PathBuf::from("/tmp/packs/mypack/actions/hello.py")),
runtime_name: None,
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024,
max_stderr_bytes: 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -899,6 +965,9 @@ mod tests {
code: None,
code_path: Some(PathBuf::from("/tmp/packs/mypack/actions/hello.sh")),
runtime_name: None,
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024,
max_stderr_bytes: 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -986,6 +1055,9 @@ mod tests {
code: None,
code_path: Some(script_path),
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024 * 1024,
max_stderr_bytes: 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -1018,6 +1090,7 @@ mod tests {
},
environment: None,
dependencies: None,
env_vars: HashMap::new(),
};
let runtime = ProcessRuntime::new(
@@ -1039,6 +1112,9 @@ mod tests {
code: None,
code_path: Some(script_path),
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024 * 1024,
max_stderr_bytes: 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -1074,6 +1150,9 @@ mod tests {
code: Some("echo 'inline shell code'".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024 * 1024,
max_stderr_bytes: 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -1121,6 +1200,9 @@ mod tests {
code: None,
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024 * 1024,
max_stderr_bytes: 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -1226,6 +1308,9 @@ mod tests {
code: None,
code_path: Some(script_path),
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 1024 * 1024,
max_stderr_bytes: 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),

12
crates/worker/src/runtime/python.rs
View File

@@ -665,6 +665,9 @@ def run(x, y):
),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -701,6 +704,9 @@ def run():
),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -737,6 +743,9 @@ def run():
),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -786,6 +795,9 @@ def run():
),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),

24
crates/worker/src/runtime/shell.rs
View File

@@ -615,6 +615,9 @@ mod tests {
code: Some("echo 'Hello, World!'".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -648,6 +651,9 @@ mod tests {
code: Some("echo \"Hello, $name!\"".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -676,6 +682,9 @@ mod tests {
code: Some("sleep 10".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -706,6 +715,9 @@ mod tests {
code: Some("exit 1".to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -751,6 +763,9 @@ echo "missing=$missing"
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -791,6 +806,9 @@ echo '{"id": 3, "name": "Charlie"}'
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -854,6 +872,9 @@ printf '{"status_code":200,"body":"hello","json":{\n "args": {\n "hello": "w
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),
@@ -906,6 +927,9 @@ echo '{"result": "success", "count": 42}'
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_common::models::ParameterDelivery::default(),

51
crates/worker/src/service.rs
View File

@@ -2,6 +2,16 @@
//!
//! Main service orchestration for the Attune Worker Service.
//! Manages worker registration, heartbeat, message consumption, and action execution.
//!
//! ## Startup Sequence
//!
//! 1. Connect to database and message queue
//! 2. Load runtimes from database → create `ProcessRuntime` instances
//! 3. Register worker and set up MQ infrastructure
//! 4. **Verify runtime versions** — run verification commands for each registered
//! `RuntimeVersion` to determine which are available on this host/container
//! 5. **Set up runtime environments** — create per-version environments for packs
//! 6. Start heartbeat, execution consumer, and pack registration consumer
use attune_common::config::Config;
use attune_common::db::Database;
@@ -13,6 +23,7 @@ use attune_common::mq::{
PackRegisteredPayload, Publisher, PublisherConfig,
};
use attune_common::repositories::{execution::ExecutionRepository, FindById};
use attune_common::runtime_detection::runtime_in_filter;
use chrono::Utc;
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
@@ -34,6 +45,7 @@ use crate::runtime::process::ProcessRuntime;
use crate::runtime::shell::ShellRuntime;
use crate::runtime::RuntimeRegistry;
use crate::secrets::SecretManager;
use crate::version_verify;
use attune_common::repositories::runtime::RuntimeRepository;
use attune_common::repositories::List;
@@ -187,9 +199,11 @@ impl WorkerService {
for rt in executable_runtimes {
let rt_name = rt.name.to_lowercase();
// Apply filter if ATTUNE_WORKER_RUNTIMES is set
// Apply filter if ATTUNE_WORKER_RUNTIMES is set.
// Uses alias-aware matching so that e.g. filter "node"
// matches DB runtime name "Node.js" (lowercased to "node.js").
if let Some(ref filter) = runtime_filter {
if !filter.contains(&rt_name) {
if !runtime_in_filter(&rt_name, filter) {
debug!(
"Skipping runtime '{}' (not in ATTUNE_WORKER_RUNTIMES filter)",
rt_name
@@ -353,9 +367,15 @@ impl WorkerService {
})?;
info!("Worker-specific message queue infrastructure setup completed");
// Verify which runtime versions are available on this system.
// This updates the `available` flag in the database so that
// `select_best_version()` only considers genuinely present versions.
self.verify_runtime_versions().await;
// Proactively set up runtime environments for all registered packs.
// This runs before we start consuming execution messages so that
// environments are ready by the time the first execution arrives.
// Now version-aware: creates per-version environments where needed.
self.scan_and_setup_environments().await;
// Start heartbeat
@@ -380,6 +400,33 @@ impl WorkerService {
/// 3. Wait for in-flight tasks with timeout
/// 4. Close MQ connection
/// 5. Close DB connection
/// Verify which runtime versions are available on this host/container.
///
/// Runs each version's verification commands (from `distributions` JSONB)
/// and updates the `available` flag in the database. This ensures that
/// `select_best_version()` only considers versions whose interpreters
/// are genuinely present.
async fn verify_runtime_versions(&self) {
let filter_refs: Option<Vec<String>> = self.runtime_filter.clone();
let filter_slice: Option<&[String]> = filter_refs.as_deref();
let result = version_verify::verify_all_runtime_versions(&self.db_pool, filter_slice).await;
if !result.errors.is_empty() {
warn!(
"Runtime version verification completed with {} error(s): {:?}",
result.errors.len(),
result.errors,
);
} else {
info!(
"Runtime version verification complete: {} checked, \
{} available, {} unavailable",
result.total_checked, result.available, result.unavailable,
);
}
}
/// Scan all registered packs and create missing runtime environments.
async fn scan_and_setup_environments(&self) {
let filter_refs: Option<Vec<String>> = self.runtime_filter.clone();

485
crates/worker/src/version_verify.rs Normal file
View File

@@ -0,0 +1,485 @@
//! Runtime Version Verification
//!
//! At worker startup, this module verifies which runtime versions are actually
//! available on the system by running each version's verification commands
//! (from the `distributions` JSONB column). Versions that pass verification
//! are marked `available = true`; those that fail are marked `available = false`.
//!
//! This ensures the worker has an accurate picture of what it can execute,
//! and `select_best_version()` only considers versions whose interpreters
//! are genuinely present on this particular host/container.
use attune_common::repositories::List;
use sqlx::PgPool;
use std::time::Duration;
use tokio::process::Command;
use tracing::{debug, info, warn};
use attune_common::models::RuntimeVersion;
use attune_common::repositories::runtime_version::RuntimeVersionRepository;
use attune_common::runtime_detection::runtime_in_filter;
/// Result of verifying all runtime versions at startup.
#[derive(Debug)]
pub struct VersionVerificationResult {
/// Total number of versions checked.
pub total_checked: usize,
/// Number of versions marked as available.
pub available: usize,
/// Number of versions marked as unavailable.
pub unavailable: usize,
/// Errors encountered during verification (non-fatal).
pub errors: Vec<String>,
}
/// A single verification command extracted from the `distributions` JSONB.
#[derive(Debug)]
struct VerificationCommand {
binary: String,
args: Vec<String>,
expected_exit_code: i32,
pattern: Option<String>,
#[allow(dead_code)]
priority: i32,
}
/// Verify all registered runtime versions and update their `available` flag.
///
/// For each `RuntimeVersion` row in the database:
/// 1. Extract verification commands from `distributions.verification.commands`
/// 2. Run each command (in priority order) until one succeeds
/// 3. Update `available` and `verified_at` in the database
///
/// # Arguments
/// * `pool` - Database connection pool
/// * `runtime_filter` - Optional runtime name filter (from `ATTUNE_WORKER_RUNTIMES`)
pub async fn verify_all_runtime_versions(
pool: &PgPool,
runtime_filter: Option<&[String]>,
) -> VersionVerificationResult {
info!("Starting runtime version verification");
let mut result = VersionVerificationResult {
total_checked: 0,
available: 0,
unavailable: 0,
errors: Vec::new(),
};
// Load all runtime versions
let versions: Vec<RuntimeVersion> = match RuntimeVersionRepository::list(pool).await {
Ok(v) => v,
Err(e) => {
let msg = format!("Failed to load runtime versions from database: {}", e);
warn!("{}", msg);
result.errors.push(msg);
return result;
}
};
if versions.is_empty() {
debug!("No runtime versions registered, skipping verification");
return result;
}
info!("Found {} runtime version(s) to verify", versions.len());
for version in &versions {
// Apply runtime filter: extract the runtime base name from the ref
// e.g., "core.python" → "python"
let rt_base_name = version
.runtime_ref
.split('.')
.last()
.unwrap_or(&version.runtime_ref)
.to_lowercase();
if let Some(filter) = runtime_filter {
if !runtime_in_filter(&rt_base_name, filter) {
debug!(
"Skipping version '{}' of runtime '{}' (not in worker runtime filter)",
version.version, version.runtime_ref,
);
continue;
}
}
result.total_checked += 1;
let is_available = verify_single_version(version).await;
// Update the database
match RuntimeVersionRepository::set_availability(pool, version.id, is_available).await {
Ok(_) => {
if is_available {
info!(
"Runtime version '{}' {} is available",
version.runtime_ref, version.version,
);
result.available += 1;
} else {
info!(
"Runtime version '{}' {} is NOT available on this system",
version.runtime_ref, version.version,
);
result.unavailable += 1;
}
}
Err(e) => {
let msg = format!(
"Failed to update availability for version '{}' {}: {}",
version.runtime_ref, version.version, e,
);
warn!("{}", msg);
result.errors.push(msg);
}
}
}
info!(
"Runtime version verification complete: {} checked, {} available, {} unavailable, {} error(s)",
result.total_checked,
result.available,
result.unavailable,
result.errors.len(),
);
result
}
/// Verify a single runtime version by running its verification commands.
///
/// Returns `true` if at least one verification command succeeds.
async fn verify_single_version(version: &RuntimeVersion) -> bool {
let commands = extract_verification_commands(&version.distributions);
if commands.is_empty() {
// No verification commands — try using the version's execution_config
// interpreter binary with --version as a basic check.
let exec_config = version.parsed_execution_config();
let binary = &exec_config.interpreter.binary;
if binary.is_empty() {
debug!(
"No verification commands and no interpreter for '{}' {}. \
Assuming available (will fail at execution time if not).",
version.runtime_ref, version.version,
);
return true;
}
debug!(
"No verification commands for '{}' {}. \
Falling back to '{} --version' check.",
version.runtime_ref, version.version, binary,
);
return run_basic_binary_check(binary).await;
}
// Run commands in priority order (lowest priority number = highest priority)
for cmd in &commands {
match run_verification_command(cmd).await {
Ok(true) => {
debug!(
"Verification passed for '{}' {} using binary '{}'",
version.runtime_ref, version.version, cmd.binary,
);
return true;
}
Ok(false) => {
debug!(
"Verification failed for '{}' {} using binary '{}' \
(pattern mismatch or non-zero exit)",
version.runtime_ref, version.version, cmd.binary,
);
}
Err(e) => {
debug!(
"Verification command '{}' for '{}' {} failed: {}",
cmd.binary, version.runtime_ref, version.version, e,
);
}
}
}
false
}
/// Extract verification commands from the `distributions` JSONB.
///
/// Expected structure:
/// ```json
/// {
/// "verification": {
/// "commands": [
/// {
/// "binary": "python3.12",
/// "args": ["--version"],
/// "exit_code": 0,
/// "pattern": "Python 3\\.12\\.",
/// "priority": 1
/// }
/// ]
/// }
/// }
/// ```
fn extract_verification_commands(distributions: &serde_json::Value) -> Vec<VerificationCommand> {
let mut commands = Vec::new();
let cmds = match distributions
.get("verification")
.and_then(|v| v.get("commands"))
.and_then(|v| v.as_array())
{
Some(arr) => arr,
None => return commands,
};
for cmd_val in cmds {
let binary = match cmd_val.get("binary").and_then(|v| v.as_str()) {
Some(b) => b.to_string(),
None => continue,
};
let args: Vec<String> = cmd_val
.get("args")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default();
let expected_exit_code = cmd_val
.get("exit_code")
.and_then(|v| v.as_i64())
.unwrap_or(0) as i32;
let pattern = cmd_val
.get("pattern")
.and_then(|v| v.as_str())
.map(String::from);
let priority = cmd_val
.get("priority")
.and_then(|v| v.as_i64())
.unwrap_or(100) as i32;
commands.push(VerificationCommand {
binary,
args,
expected_exit_code,
pattern,
priority,
});
}
// Sort by priority (lowest number = highest priority)
commands.sort_by_key(|c| c.priority);
commands
}
/// Run a single verification command and check exit code + output pattern.
async fn run_verification_command(cmd: &VerificationCommand) -> std::result::Result<bool, String> {
let output = Command::new(&cmd.binary)
.args(&cmd.args)
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true)
.spawn()
.map_err(|e| format!("Failed to spawn '{}': {}", cmd.binary, e))?;
let result = tokio::time::timeout(Duration::from_secs(10), output.wait_with_output())
.await
.map_err(|_| format!("Verification command '{}' timed out after 10s", cmd.binary))?
.map_err(|e| format!("Failed to wait for '{}': {}", cmd.binary, e))?;
// Check exit code
let actual_exit = result.status.code().unwrap_or(-1);
if actual_exit != cmd.expected_exit_code {
return Ok(false);
}
// Check output pattern if specified
if let Some(ref pattern) = cmd.pattern {
let stdout = String::from_utf8_lossy(&result.stdout);
let stderr = String::from_utf8_lossy(&result.stderr);
let combined = format!("{}{}", stdout, stderr);
let re = regex::Regex::new(pattern)
.map_err(|e| format!("Invalid verification pattern '{}': {}", pattern, e))?;
if !re.is_match(&combined) {
debug!(
"Pattern '{}' did not match output of '{}': stdout='{}', stderr='{}'",
pattern,
cmd.binary,
stdout.trim(),
stderr.trim(),
);
return Ok(false);
}
}
Ok(true)
}
/// Basic binary availability check: run `binary --version` and check for exit 0.
async fn run_basic_binary_check(binary: &str) -> bool {
match Command::new(binary)
.arg("--version")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true)
.spawn()
{
Ok(child) => {
match tokio::time::timeout(Duration::from_secs(10), child.wait_with_output()).await {
Ok(Ok(output)) => output.status.success(),
Ok(Err(e)) => {
debug!("Binary check for '{}' failed: {}", binary, e);
false
}
Err(_) => {
debug!("Binary check for '{}' timed out", binary);
false
}
}
}
Err(e) => {
debug!("Failed to spawn '{}': {}", binary, e);
false
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_extract_verification_commands_full() {
let distributions = json!({
"verification": {
"commands": [
{
"binary": "python3.12",
"args": ["--version"],
"exit_code": 0,
"pattern": "Python 3\\.12\\.",
"priority": 1
},
{
"binary": "python3",
"args": ["--version"],
"exit_code": 0,
"pattern": "Python 3\\.12\\.",
"priority": 2
}
]
}
});
let cmds = extract_verification_commands(&distributions);
assert_eq!(cmds.len(), 2);
assert_eq!(cmds[0].binary, "python3.12");
assert_eq!(cmds[0].priority, 1);
assert_eq!(cmds[1].binary, "python3");
assert_eq!(cmds[1].priority, 2);
assert_eq!(cmds[0].args, vec!["--version"]);
assert_eq!(cmds[0].expected_exit_code, 0);
assert_eq!(cmds[0].pattern.as_deref(), Some("Python 3\\.12\\."));
}
#[test]
fn test_extract_verification_commands_empty() {
let distributions = json!({});
let cmds = extract_verification_commands(&distributions);
assert!(cmds.is_empty());
}
#[test]
fn test_extract_verification_commands_no_commands_array() {
let distributions = json!({
"verification": {}
});
let cmds = extract_verification_commands(&distributions);
assert!(cmds.is_empty());
}
#[test]
fn test_extract_verification_commands_missing_binary() {
let distributions = json!({
"verification": {
"commands": [
{
"args": ["--version"],
"exit_code": 0
}
]
}
});
let cmds = extract_verification_commands(&distributions);
assert!(cmds.is_empty(), "Commands without binary should be skipped");
}
#[test]
fn test_extract_verification_commands_defaults() {
let distributions = json!({
"verification": {
"commands": [
{
"binary": "node"
}
]
}
});
let cmds = extract_verification_commands(&distributions);
assert_eq!(cmds.len(), 1);
assert_eq!(cmds[0].binary, "node");
assert!(cmds[0].args.is_empty());
assert_eq!(cmds[0].expected_exit_code, 0);
assert!(cmds[0].pattern.is_none());
assert_eq!(cmds[0].priority, 100);
}
#[test]
fn test_extract_verification_commands_sorted_by_priority() {
let distributions = json!({
"verification": {
"commands": [
{ "binary": "low", "priority": 10 },
{ "binary": "high", "priority": 1 },
{ "binary": "mid", "priority": 5 }
]
}
});
let cmds = extract_verification_commands(&distributions);
assert_eq!(cmds.len(), 3);
assert_eq!(cmds[0].binary, "high");
assert_eq!(cmds[1].binary, "mid");
assert_eq!(cmds[2].binary, "low");
}
#[tokio::test]
async fn test_run_basic_binary_check_nonexistent() {
// A binary that definitely doesn't exist
let result = run_basic_binary_check("__nonexistent_binary_12345__").await;
assert!(!result);
}
#[tokio::test]
async fn test_run_verification_command_nonexistent() {
let cmd = VerificationCommand {
binary: "__nonexistent_binary_12345__".to_string(),
args: vec!["--version".to_string()],
expected_exit_code: 0,
pattern: None,
priority: 1,
};
let result = run_verification_command(&cmd).await;
assert!(result.is_err());
}
}

25
crates/worker/tests/dependency_isolation_test.rs
View File

@@ -48,6 +48,7 @@ fn make_python_config() -> RuntimeExecutionConfig {
"{manifest_path}".to_string(),
],
}),
env_vars: std::collections::HashMap::new(),
}
}
@@ -60,6 +61,7 @@ fn make_shell_config() -> RuntimeExecutionConfig {
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
}
}
@@ -76,6 +78,9 @@ fn make_context(action_ref: &str, entry_point: &str, runtime_name: &str) -> Exec
code: None,
code_path: None,
runtime_name: Some(runtime_name.to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: ParameterDelivery::default(),
@@ -108,7 +113,10 @@ async fn test_python_venv_creation_via_process_runtime() {
.expect("Failed to create venv environment");
// Verify venv was created at the external runtime_envs location
assert!(env_dir.exists(), "Virtualenv directory should exist at external location");
assert!(
env_dir.exists(),
"Virtualenv directory should exist at external location"
);
let venv_python = env_dir.join("bin").join("python3");
assert!(
@@ -319,11 +327,20 @@ async fn test_multiple_pack_isolation() {
// Each pack should have its own venv at the external location
assert!(env_dir_a.exists(), "pack_a should have its own venv");
assert!(env_dir_b.exists(), "pack_b should have its own venv");
assert_ne!(env_dir_a, env_dir_b, "Venvs should be in different directories");
assert_ne!(
env_dir_a, env_dir_b,
"Venvs should be in different directories"
);
// Pack directories should remain clean
assert!(!pack_a_dir.join(".venv").exists(), "pack_a dir should not contain .venv");
assert!(!pack_b_dir.join(".venv").exists(), "pack_b dir should not contain .venv");
assert!(
!pack_a_dir.join(".venv").exists(),
"pack_a dir should not contain .venv"
);
assert!(
!pack_b_dir.join(".venv").exists(),
"pack_b dir should not contain .venv"
);
}
#[tokio::test]

11
crates/worker/tests/log_truncation_test.rs
View File

@@ -19,6 +19,7 @@ fn make_python_process_runtime(packs_base_dir: PathBuf) -> ProcessRuntime {
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
ProcessRuntime::new("python".to_string(), config, packs_base_dir.clone(), packs_base_dir.join("../runtime_envs"))
}
@@ -42,6 +43,9 @@ fn make_python_context(
code: Some(code.to_string()),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes,
max_stderr_bytes,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -121,6 +125,9 @@ done
code: Some(code.to_string()),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 400, // Small limit
max_stderr_bytes: 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -258,6 +265,7 @@ async fn test_shell_process_runtime_truncation() {
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
let runtime = ProcessRuntime::new("shell".to_string(), config, tmp.path().to_path_buf(), tmp.path().join("runtime_envs"));
@@ -275,6 +283,9 @@ async fn test_shell_process_runtime_truncation() {
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 500,
max_stderr_bytes: 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),

27
crates/worker/tests/security_tests.rs
View File

@@ -20,6 +20,7 @@ fn make_python_process_runtime(packs_base_dir: PathBuf) -> ProcessRuntime {
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
let runtime_envs_dir = packs_base_dir.parent().unwrap_or(&packs_base_dir).join("runtime_envs");
ProcessRuntime::new("python".to_string(), config, packs_base_dir, runtime_envs_dir)
@@ -68,6 +69,9 @@ print(json.dumps(result))
code: Some(code.to_string()),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -158,6 +162,9 @@ echo "SECURITY_PASS: Secrets not in environment but accessible via get_secret"
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -219,6 +226,9 @@ print(json.dumps({'secret_a': secrets.get('secret_a')}))
code: Some(code1.to_string()),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -261,6 +271,9 @@ print(json.dumps({
code: Some(code2.to_string()),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -312,6 +325,9 @@ print("ok")
code: Some(code.to_string()),
code_path: None,
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -360,6 +376,9 @@ fi
),
code_path: None,
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -409,6 +428,7 @@ echo "PASS: No secrets in environment"
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
let runtime = ProcessRuntime::new("shell".to_string(), config, tmp.path().to_path_buf(), tmp.path().join("runtime_envs"));
@@ -428,6 +448,9 @@ echo "PASS: No secrets in environment"
code: None,
code_path: Some(actions_dir.join("check_env.sh")),
runtime_name: Some("shell".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),
@@ -476,6 +499,7 @@ print(json.dumps({"leaked": leaked}))
},
environment: None,
dependencies: None,
env_vars: std::collections::HashMap::new(),
};
let runtime = ProcessRuntime::new("python".to_string(), config, tmp.path().to_path_buf(), tmp.path().join("runtime_envs"));
@@ -495,6 +519,9 @@ print(json.dumps({"leaked": leaked}))
code: None,
code_path: Some(actions_dir.join("check_env.py")),
runtime_name: Some("python".to_string()),
runtime_config_override: None,
runtime_env_dir_suffix: None,
selected_runtime_version: None,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
parameter_delivery: attune_worker::runtime::ParameterDelivery::default(),