attune-system/attune
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

node running, runtime version awareness

Browse Source
This commit is contained in:
David Culbreth
2026-02-25 23:24:07 -06:00
parent e89b5991ec
commit 495b81236a
54 changed files with 4308 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

207
crates/worker/src/env_setup.rs
View File

@@ -8,6 +8,19 @@
//! The goal is to ensure environments are ready *before* the first execution,
//! eliminating the first-run penalty and potential permission errors that occur
//! when setup is deferred to execution time.
//!
//! ## Version-Aware Environments
//!
//! When runtime versions are registered (e.g., Python 3.11, 3.12, 3.13), this
//! module creates per-version environments at:
//! `{runtime_envs_dir}/{pack_ref}/{runtime_name}-{version}`
//!
//! For example: `/opt/attune/runtime_envs/my_pack/python-3.12`
//!
//! This ensures that different versions maintain isolated environments with
//! their own interpreter binaries and installed dependencies. A base (unversioned)
//! environment is also created for backward compatibility with actions that don't
//! declare a version constraint.
use std::collections::{HashMap, HashSet};
use std::path::Path;
@@ -15,11 +28,14 @@ use std::path::Path;
use sqlx::PgPool;
use tracing::{debug, error, info, warn};
use attune_common::models::RuntimeVersion;
use attune_common::mq::PackRegisteredPayload;
use attune_common::repositories::action::ActionRepository;
use attune_common::repositories::pack::PackRepository;
use attune_common::repositories::runtime::RuntimeRepository;
use attune_common::repositories::runtime_version::RuntimeVersionRepository;
use attune_common::repositories::{FindById, List};
use attune_common::runtime_detection::runtime_in_filter;
// Re-export the utility that the API also uses so callers can reach it from
// either crate without adding a direct common dependency for this one function.
@@ -96,6 +112,26 @@ pub async fn scan_and_setup_all_environments(
}
};
// Load all runtime versions, indexed by runtime ID
let version_map: HashMap<i64, Vec<RuntimeVersion>> =
match RuntimeVersionRepository::list(db_pool).await {
Ok(versions) => {
let mut map: HashMap<i64, Vec<RuntimeVersion>> = HashMap::new();
for v in versions {
map.entry(v.runtime).or_default().push(v);
}
map
}
Err(e) => {
warn!(
"Failed to load runtime versions from database: {}. \
Version-specific environments will not be created.",
e
);
HashMap::new()
}
};
info!("Found {} registered pack(s) to scan", packs.len());
for pack in &packs {
@@ -109,6 +145,7 @@ pub async fn scan_and_setup_all_environments(
packs_base_dir,
runtime_envs_dir,
&runtime_map,
&version_map,
)
.await;
@@ -164,13 +201,13 @@ pub async fn setup_environments_for_registered_pack(
return pack_result;
}
// Filter to runtimes this worker supports
// Filter to runtimes this worker supports (alias-aware matching)
let target_runtimes: Vec<&String> = event
.runtime_names
.iter()
.filter(|name| {
if let Some(filter) = runtime_filter {
filter.contains(name)
runtime_in_filter(name, filter)
} else {
true
}
@@ -219,6 +256,7 @@ pub async fn setup_environments_for_registered_pack(
continue;
}
// Set up base (unversioned) environment
let env_dir = runtime_envs_dir.join(&event.pack_ref).join(rt_name);
let process_runtime = ProcessRuntime::new(
@@ -248,6 +286,19 @@ pub async fn setup_environments_for_registered_pack(
pack_result.errors.push(msg);
}
}
// Set up per-version environments for available runtime versions
setup_version_environments(
db_pool,
rt.id,
rt_name,
&event.pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
}
pack_result
@@ -256,7 +307,8 @@ pub async fn setup_environments_for_registered_pack(
/// Internal helper: set up environments for a single pack during the startup scan.
///
/// Discovers which runtimes the pack's actions use, filters by this worker's
/// capabilities, and creates any missing environments.
/// capabilities, and creates any missing environments. Also creates per-version
/// environments for runtimes that have registered versions.
#[allow(clippy::too_many_arguments)]
async fn setup_environments_for_pack(
db_pool: &PgPool,
@@ -266,6 +318,7 @@ async fn setup_environments_for_pack(
packs_base_dir: &Path,
runtime_envs_dir: &Path,
runtime_map: &HashMap<i64, attune_common::models::Runtime>,
version_map: &HashMap<i64, Vec<RuntimeVersion>>,
) -> PackEnvSetupResult {
let mut pack_result = PackEnvSetupResult {
pack_ref: pack_ref.to_string(),
@@ -327,6 +380,25 @@ async fn setup_environments_for_pack(
&mut pack_result,
)
.await;
// Also set up version-specific environments
let versions = match RuntimeVersionRepository::find_available_by_runtime(
db_pool, runtime_id,
)
.await
{
Ok(v) => v,
Err(_) => Vec::new(),
};
setup_version_environments_from_list(
&versions,
&rt_name,
pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
continue;
}
Ok(None) => {
@@ -353,6 +425,22 @@ async fn setup_environments_for_pack(
&mut pack_result,
)
.await;
// Set up per-version environments for available versions of this runtime
if let Some(versions) = version_map.get(&runtime_id) {
let available_versions: Vec<RuntimeVersion> =
versions.iter().filter(|v| v.available).cloned().collect();
setup_version_environments_from_list(
&available_versions,
&rt_name,
pack_ref,
&pack_dir,
packs_base_dir,
runtime_envs_dir,
&mut pack_result,
)
.await;
}
}
if !pack_result.environments_created.is_empty() {
@@ -377,9 +465,9 @@ async fn process_runtime_for_pack(
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
// Apply worker runtime filter
// Apply worker runtime filter (alias-aware matching)
if let Some(filter) = runtime_filter {
if !filter.iter().any(|f| f == rt_name) {
if !runtime_in_filter(rt_name, filter) {
debug!(
"Runtime '{}' not in worker filter, skipping for pack '{}'",
rt_name, pack_ref,
@@ -430,6 +518,115 @@ async fn process_runtime_for_pack(
}
}
/// Set up per-version environments for a runtime, given a list of available versions.
///
/// For each available version, creates an environment at:
/// `{runtime_envs_dir}/{pack_ref}/{runtime_name}-{version}`
///
/// This uses the version's own `execution_config` (which may specify a different
/// interpreter binary, environment create command, etc.).
#[allow(clippy::too_many_arguments)]
async fn setup_version_environments_from_list(
versions: &[RuntimeVersion],
rt_name: &str,
pack_ref: &str,
pack_dir: &Path,
packs_base_dir: &Path,
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
if versions.is_empty() {
return;
}
for version in versions {
let version_exec_config = version.parsed_execution_config();
// Skip versions with no environment config and no dependencies
if version_exec_config.environment.is_none()
&& !version_exec_config.has_dependencies(pack_dir)
{
debug!(
"Version '{}' {} has no environment config, skipping for pack '{}'",
version.runtime_ref, version.version, pack_ref,
);
continue;
}
let version_env_suffix = format!("{}-{}", rt_name, version.version);
let version_env_dir = runtime_envs_dir.join(pack_ref).join(&version_env_suffix);
let version_runtime = ProcessRuntime::new(
rt_name.to_string(),
version_exec_config,
packs_base_dir.to_path_buf(),
runtime_envs_dir.to_path_buf(),
);
match version_runtime
.setup_pack_environment(pack_dir, &version_env_dir)
.await
{
Ok(()) => {
info!(
"Version environment '{}' ready for pack '{}'",
version_env_suffix, pack_ref,
);
pack_result.environments_created.push(version_env_suffix);
}
Err(e) => {
let msg = format!(
"Failed to set up version environment '{}' for pack '{}': {}",
version_env_suffix, pack_ref, e,
);
warn!("{}", msg);
pack_result.errors.push(msg);
}
}
}
}
/// Set up per-version environments for a runtime by querying the database.
///
/// This is a convenience wrapper around `setup_version_environments_from_list`
/// that queries available versions from the database first. Used in the
/// pack.registered event handler where we don't have a pre-loaded version map.
#[allow(clippy::too_many_arguments)]
async fn setup_version_environments(
db_pool: &PgPool,
runtime_id: i64,
rt_name: &str,
pack_ref: &str,
pack_dir: &Path,
packs_base_dir: &Path,
runtime_envs_dir: &Path,
pack_result: &mut PackEnvSetupResult,
) {
let versions =
match RuntimeVersionRepository::find_available_by_runtime(db_pool, runtime_id).await {
Ok(v) => v,
Err(e) => {
debug!(
"Failed to load versions for runtime '{}' (id {}): {}. \
Skipping version-specific environments.",
rt_name, runtime_id, e,
);
return;
}
};
setup_version_environments_from_list(
&versions,
rt_name,
pack_ref,
pack_dir,
packs_base_dir,
runtime_envs_dir,
pack_result,
)
.await;
}
/// Determine the runtime filter from the `ATTUNE_WORKER_RUNTIMES` environment variable.
///
/// Returns `None` if the variable is not set (meaning all runtimes are accepted).