attune-system/attune
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[wip] universal workers

Browse Source
This commit is contained in:
David Culbreth
2026-03-21 07:32:11 -05:00
parent 0782675a2b
commit 8ba7e3bb84
59 changed files with 4971 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
crates/api/src/openapi.rs
View File

@@ -199,6 +199,10 @@ use crate::dto::{
crate::routes::webhooks::disable_webhook,
crate::routes::webhooks::regenerate_webhook_key,
crate::routes::webhooks::receive_webhook,
// Agent
crate::routes::agent::download_agent_binary,
crate::routes::agent::agent_info,
),
components(
schemas(
@@ -341,6 +345,10 @@ use crate::dto::{
WebhookReceiverRequest,
WebhookReceiverResponse,
ApiResponse<WebhookReceiverResponse>,
// Agent DTOs
crate::routes::agent::AgentBinaryInfo,
crate::routes::agent::AgentArchInfo,
)
),
modifiers(&SecurityAddon),
@@ -359,6 +367,7 @@ use crate::dto::{
(name = "secrets", description = "Secret management endpoints"),
(name = "workflows", description = "Workflow management endpoints"),
(name = "webhooks", description = "Webhook management and receiver endpoints"),
(name = "agent", description = "Agent binary distribution endpoints"),
)
)]
pub struct ApiDoc;
@@ -441,14 +450,14 @@ mod tests {
// We have 57 unique paths with 81 total operations (HTTP methods)
// This test ensures we don't accidentally remove endpoints
assert!(
path_count >= 57,
"Expected at least 57 unique API paths, found {}",
path_count >= 59,
"Expected at least 59 unique API paths, found {}",
path_count
);
assert!(
operation_count >= 81,
"Expected at least 81 API operations, found {}",
operation_count >= 83,
"Expected at least 83 API operations, found {}",
operation_count
);

304
crates/api/src/routes/agent.rs Normal file
View File

@@ -0,0 +1,304 @@
//! Agent binary download endpoints
//!
//! Provides endpoints for downloading the attune-agent binary for injection
//! into arbitrary containers. This supports deployments where shared Docker
//! volumes are impractical (Kubernetes, ECS, remote Docker hosts).
use axum::{
body::Body,
extract::{Query, State},
http::{header, HeaderMap, StatusCode},
response::IntoResponse,
routing::get,
Json, Router,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::fs;
use tokio_util::io::ReaderStream;
use utoipa::{IntoParams, ToSchema};
use crate::state::AppState;
/// Query parameters for the binary download endpoint
#[derive(Debug, Deserialize, IntoParams)]
pub struct BinaryDownloadParams {
/// Target architecture (x86_64, aarch64). Defaults to x86_64.
#[param(example = "x86_64")]
pub arch: Option<String>,
/// Optional bootstrap token for authentication
pub token: Option<String>,
}
/// Agent binary metadata
#[derive(Debug, Serialize, ToSchema)]
pub struct AgentBinaryInfo {
/// Available architectures
pub architectures: Vec<AgentArchInfo>,
/// Agent version (from build)
pub version: String,
}
/// Per-architecture binary info
#[derive(Debug, Serialize, ToSchema)]
pub struct AgentArchInfo {
/// Architecture name
pub arch: String,
/// Binary size in bytes
pub size_bytes: u64,
/// Whether this binary is available
pub available: bool,
}
/// Validate that the architecture name is safe (no path traversal) and normalize it.
fn validate_arch(arch: &str) -> Result<&str, (StatusCode, Json<serde_json::Value>)> {
match arch {
"x86_64" | "aarch64" => Ok(arch),
// Accept arm64 as an alias for aarch64
"arm64" => Ok("aarch64"),
_ => Err((
StatusCode::BAD_REQUEST,
Json(serde_json::json!({
"error": "Invalid architecture",
"message": format!("Unsupported architecture '{}'. Supported: x86_64, aarch64", arch),
})),
)),
}
}
/// Validate bootstrap token if configured.
///
/// If the agent config has a `bootstrap_token` set, the request must provide it
/// via the `X-Agent-Token` header or the `token` query parameter. If no token
/// is configured, access is unrestricted.
fn validate_token(
config: &attune_common::config::Config,
headers: &HeaderMap,
query_token: &Option<String>,
) -> Result<(), (StatusCode, Json<serde_json::Value>)> {
let expected_token = config
.agent
.as_ref()
.and_then(|ac| ac.bootstrap_token.as_ref());
let expected_token = match expected_token {
Some(t) => t,
None => return Ok(()), // No token configured, allow access
};
// Check X-Agent-Token header first, then query param
let provided_token = headers
.get("x-agent-token")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.or_else(|| query_token.clone());
match provided_token {
Some(ref t) if t == expected_token => Ok(()),
Some(_) => Err((
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Invalid token",
"message": "The provided bootstrap token is invalid",
})),
)),
None => Err((
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Token required",
"message": "A bootstrap token is required. Provide via X-Agent-Token header or token query parameter.",
})),
)),
}
}
/// Download the agent binary
///
/// Returns the statically-linked attune-agent binary for the requested architecture.
/// The binary can be injected into any container to turn it into an Attune worker.
#[utoipa::path(
get,
path = "/api/v1/agent/binary",
params(BinaryDownloadParams),
responses(
(status = 200, description = "Agent binary", content_type = "application/octet-stream"),
(status = 400, description = "Invalid architecture"),
(status = 401, description = "Invalid or missing bootstrap token"),
(status = 404, description = "Agent binary not found"),
(status = 503, description = "Agent binary distribution not configured"),
),
tag = "agent"
)]
pub async fn download_agent_binary(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Query(params): Query<BinaryDownloadParams>,
) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
// Validate bootstrap token if configured
validate_token(&state.config, &headers, &params.token)?;
let agent_config = state.config.agent.as_ref().ok_or_else(|| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(serde_json::json!({
"error": "Not configured",
"message": "Agent binary distribution is not configured. Set agent.binary_dir in config.",
})),
)
})?;
let arch = params.arch.as_deref().unwrap_or("x86_64");
let arch = validate_arch(arch)?;
let binary_dir = std::path::Path::new(&agent_config.binary_dir);
// Try arch-specific binary first, then fall back to generic name
let arch_specific = binary_dir.join(format!("attune-agent-{}", arch));
let generic = binary_dir.join("attune-agent");
let binary_path = if arch_specific.exists() {
arch_specific
} else if generic.exists() {
tracing::debug!(
"Arch-specific binary not found at {:?}, falling back to {:?}",
arch_specific,
generic
);
generic
} else {
tracing::warn!(
"Agent binary not found. Checked: {:?} and {:?}",
arch_specific,
generic
);
return Err((
StatusCode::NOT_FOUND,
Json(serde_json::json!({
"error": "Not found",
"message": format!(
"Agent binary not found for architecture '{}'. Ensure the agent binary is built and placed in '{}'.",
arch,
agent_config.binary_dir
),
})),
));
};
// Get file metadata for Content-Length
let metadata = fs::metadata(&binary_path).await.map_err(|e| {
tracing::error!(
"Failed to read agent binary metadata at {:?}: {}",
binary_path,
e
);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": "Internal error",
"message": "Failed to read agent binary",
})),
)
})?;
// Open file for streaming
let file = fs::File::open(&binary_path).await.map_err(|e| {
tracing::error!("Failed to open agent binary at {:?}: {}", binary_path, e);
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": "Internal error",
"message": "Failed to open agent binary",
})),
)
})?;
let stream = ReaderStream::new(file);
let body = Body::from_stream(stream);
let headers_response = [
(header::CONTENT_TYPE, "application/octet-stream".to_string()),
(
header::CONTENT_DISPOSITION,
"attachment; filename=\"attune-agent\"".to_string(),
),
(header::CONTENT_LENGTH, metadata.len().to_string()),
(header::CACHE_CONTROL, "public, max-age=3600".to_string()),
];
tracing::info!(
arch = arch,
size_bytes = metadata.len(),
path = ?binary_path,
"Serving agent binary download"
);
Ok((headers_response, body))
}
/// Get agent binary metadata
///
/// Returns information about available agent binaries, including
/// supported architectures and binary sizes.
#[utoipa::path(
get,
path = "/api/v1/agent/info",
responses(
(status = 200, description = "Agent binary info", body = AgentBinaryInfo),
(status = 503, description = "Agent binary distribution not configured"),
),
tag = "agent"
)]
pub async fn agent_info(
State(state): State<Arc<AppState>>,
) -> Result<impl IntoResponse, (StatusCode, Json<serde_json::Value>)> {
let agent_config = state.config.agent.as_ref().ok_or_else(|| {
(
StatusCode::SERVICE_UNAVAILABLE,
Json(serde_json::json!({
"error": "Not configured",
"message": "Agent binary distribution is not configured.",
})),
)
})?;
let binary_dir = std::path::Path::new(&agent_config.binary_dir);
let architectures = ["x86_64", "aarch64"];
let mut arch_infos = Vec::new();
for arch in &architectures {
let arch_specific = binary_dir.join(format!("attune-agent-{}", arch));
let generic = binary_dir.join("attune-agent");
let (available, size_bytes) = if arch_specific.exists() {
match fs::metadata(&arch_specific).await {
Ok(m) => (true, m.len()),
Err(_) => (false, 0),
}
} else if generic.exists() {
match fs::metadata(&generic).await {
Ok(m) => (true, m.len()),
Err(_) => (false, 0),
}
} else {
(false, 0)
};
arch_infos.push(AgentArchInfo {
arch: arch.to_string(),
size_bytes,
available,
});
}
Ok(Json(AgentBinaryInfo {
architectures: arch_infos,
version: env!("CARGO_PKG_VERSION").to_string(),
}))
}
/// Create agent routes
pub fn routes() -> Router<Arc<AppState>> {
Router::new()
.route("/agent/binary", get(download_agent_binary))
.route("/agent/info", get(agent_info))
}

2
crates/api/src/routes/mod.rs
View File

@@ -1,6 +1,7 @@
//! API route modules
pub mod actions;
pub mod agent;
pub mod analytics;
pub mod artifacts;
pub mod auth;
@@ -19,6 +20,7 @@ pub mod webhooks;
pub mod workflows;
pub use actions::routes as action_routes;
pub use agent::routes as agent_routes;
pub use analytics::routes as analytics_routes;
pub use artifacts::routes as artifact_routes;
pub use auth::routes as auth_routes;

3
crates/api/src/routes/runtimes.rs
View File

@@ -179,6 +179,8 @@ pub async fn create_runtime(
distributions: request.distributions,
installation: request.installation,
execution_config: request.execution_config,
auto_detected: false,
detection_config: serde_json::json!({}),
},
)
.await?;
@@ -232,6 +234,7 @@ pub async fn update_runtime(
NullableJsonPatch::Clear => Patch::Clear,
}),
execution_config: request.execution_config,
..Default::default()
},
)
.await?;

1
crates/api/src/server.rs
View File

@@ -60,6 +60,7 @@ impl Server {
.merge(routes::history_routes())
.merge(routes::analytics_routes())
.merge(routes::artifact_routes())
.merge(routes::agent_routes())
.with_state(self.state.clone());
// Auth routes at root level (not versioned for frontend compatibility)

14
crates/common/src/config.rs
View File

@@ -677,6 +677,15 @@ impl Default for PackRegistryConfig {
}
}
/// Agent binary distribution configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AgentConfig {
/// Directory containing agent binary files
pub binary_dir: String,
/// Optional bootstrap token for authenticating agent binary downloads
pub bootstrap_token: Option<String>,
}
/// Executor service configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExecutorConfig {
@@ -770,6 +779,9 @@ pub struct Config {
/// Executor configuration (optional, for executor service)
pub executor: Option<ExecutorConfig>,
/// Agent configuration (optional, for agent binary distribution)
pub agent: Option<AgentConfig>,
}
fn default_service_name() -> String {
@@ -1066,6 +1078,7 @@ mod tests {
notifier: None,
pack_registry: PackRegistryConfig::default(),
executor: None,
agent: None,
};
assert_eq!(config.service_name, "attune");
@@ -1144,6 +1157,7 @@ mod tests {
notifier: None,
pack_registry: PackRegistryConfig::default(),
executor: None,
agent: None,
};
assert!(config.validate().is_ok());

2
crates/common/src/models.rs
View File

@@ -780,6 +780,8 @@ pub mod runtime {
pub installation: Option<JsonDict>,
pub installers: JsonDict,
pub execution_config: JsonDict,
pub auto_detected: bool,
pub detection_config: JsonDict,
pub created: DateTime<Utc>,
pub updated: DateTime<Utc>,
}

1
crates/common/src/pack_environment.rs
View File

@@ -374,6 +374,7 @@ impl PackEnvironmentManager {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
WHERE id = $1

3
crates/common/src/pack_registry/loader.rs
View File

@@ -418,6 +418,7 @@ impl<'a> PackComponentLoader<'a> {
None => Patch::Clear,
}),
execution_config: Some(execution_config),
..Default::default()
};
match RuntimeRepository::update(self.pool, existing.id, update_input).await {
@@ -448,6 +449,8 @@ impl<'a> PackComponentLoader<'a> {
distributions,
installation,
execution_config,
auto_detected: false,
detection_config: serde_json::json!({}),
};
match RuntimeRepository::create(self.pool, input).await {

46
crates/common/src/repositories/runtime.rs
View File

@@ -23,6 +23,13 @@ impl Repository for RuntimeRepository {
}
}
/// Columns selected for all Runtime queries. Centralised here so that
/// schema changes only need one update.
pub const SELECT_COLUMNS: &str = "id, ref, pack, pack_ref, description, name, \
distributions, installation, installers, execution_config, \
auto_detected, detection_config, \
created, updated";
/// Input for creating a new runtime
#[derive(Debug, Clone)]
pub struct CreateRuntimeInput {
@@ -34,6 +41,8 @@ pub struct CreateRuntimeInput {
pub distributions: JsonDict,
pub installation: Option<JsonDict>,
pub execution_config: JsonDict,
pub auto_detected: bool,
pub detection_config: JsonDict,
}
/// Input for updating a runtime
@@ -44,6 +53,8 @@ pub struct UpdateRuntimeInput {
pub distributions: Option<JsonDict>,
pub installation: Option<Patch<JsonDict>>,
pub execution_config: Option<JsonDict>,
pub auto_detected: Option<bool>,
pub detection_config: Option<JsonDict>,
}
#[async_trait::async_trait]
@@ -56,6 +67,7 @@ impl FindById for RuntimeRepository {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
WHERE id = $1
@@ -79,6 +91,7 @@ impl FindByRef for RuntimeRepository {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
WHERE ref = $1
@@ -102,6 +115,7 @@ impl List for RuntimeRepository {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
ORDER BY ref ASC
@@ -125,10 +139,12 @@ impl Create for RuntimeRepository {
let runtime = sqlx::query_as::<_, Runtime>(
r#"
INSERT INTO runtime (ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
distributions, installation, installers, execution_config,
auto_detected, detection_config)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
RETURNING id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
"#,
)
@@ -141,6 +157,8 @@ impl Create for RuntimeRepository {
.bind(&input.installation)
.bind(serde_json::json!({}))
.bind(&input.execution_config)
.bind(input.auto_detected)
.bind(&input.detection_config)
.fetch_one(executor)
.await?;
@@ -209,6 +227,24 @@ impl Update for RuntimeRepository {
has_updates = true;
}
if let Some(auto_detected) = input.auto_detected {
if has_updates {
query.push(", ");
}
query.push("auto_detected = ");
query.push_bind(auto_detected);
has_updates = true;
}
if let Some(detection_config) = &input.detection_config {
if has_updates {
query.push(", ");
}
query.push("detection_config = ");
query.push_bind(detection_config);
has_updates = true;
}
if !has_updates {
// No updates requested, fetch and return existing entity
return Self::get_by_id(executor, id).await;
@@ -218,7 +254,9 @@ impl Update for RuntimeRepository {
query.push_bind(id);
query.push(
" RETURNING id, ref, pack, pack_ref, description, name, \
distributions, installation, installers, execution_config, created, updated",
distributions, installation, installers, execution_config, \
auto_detected, detection_config, \
created, updated",
);
let runtime = query
@@ -255,6 +293,7 @@ impl RuntimeRepository {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
WHERE pack = $1
@@ -277,6 +316,7 @@ impl RuntimeRepository {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
WHERE LOWER(name) = LOWER($1)

43
crates/common/src/runtime_detection.rs
View File

@@ -42,6 +42,11 @@ pub fn normalize_runtime_name(name: &str) -> &str {
"python" | "python3" => "python",
"bash" | "sh" | "shell" => "shell",
"native" | "builtin" | "standalone" => "native",
"ruby" | "rb" => "ruby",
"go" | "golang" => "go",
"java" | "jdk" | "openjdk" => "java",
"perl" | "perl5" => "perl",
"r" | "rscript" => "r",
other => other,
}
}
@@ -160,6 +165,7 @@ impl RuntimeDetector {
r#"
SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime
ORDER BY ref
@@ -363,6 +369,37 @@ mod tests {
assert_eq!(normalize_runtime_name("standalone"), "native");
}
#[test]
fn test_normalize_runtime_name_ruby_variants() {
assert_eq!(normalize_runtime_name("ruby"), "ruby");
assert_eq!(normalize_runtime_name("rb"), "ruby");
}
#[test]
fn test_normalize_runtime_name_go_variants() {
assert_eq!(normalize_runtime_name("go"), "go");
assert_eq!(normalize_runtime_name("golang"), "go");
}
#[test]
fn test_normalize_runtime_name_java_variants() {
assert_eq!(normalize_runtime_name("java"), "java");
assert_eq!(normalize_runtime_name("jdk"), "java");
assert_eq!(normalize_runtime_name("openjdk"), "java");
}
#[test]
fn test_normalize_runtime_name_perl_variants() {
assert_eq!(normalize_runtime_name("perl"), "perl");
assert_eq!(normalize_runtime_name("perl5"), "perl");
}
#[test]
fn test_normalize_runtime_name_r_variants() {
assert_eq!(normalize_runtime_name("r"), "r");
assert_eq!(normalize_runtime_name("rscript"), "r");
}
#[test]
fn test_normalize_runtime_name_passthrough() {
assert_eq!(normalize_runtime_name("custom_runtime"), "custom_runtime");
@@ -390,6 +427,12 @@ mod tests {
assert!(runtime_matches_filter("python", "Python"));
assert!(runtime_matches_filter("Shell", "shell"));
assert!(runtime_matches_filter("NODEJS", "node"));
assert!(runtime_matches_filter("Ruby", "ruby"));
assert!(runtime_matches_filter("ruby", "rb"));
assert!(runtime_matches_filter("Go", "golang"));
assert!(runtime_matches_filter("R", "rscript"));
assert!(runtime_matches_filter("Java", "jdk"));
assert!(runtime_matches_filter("Perl", "perl5"));
assert!(!runtime_matches_filter("Python", "node"));
}

2
crates/common/tests/helpers.rs
View File

@@ -964,6 +964,8 @@ impl RuntimeFixture {
distributions: self.distributions,
installation: self.installation,
execution_config: self.execution_config,
auto_detected: false,
detection_config: serde_json::json!({}),
};
RuntimeRepository::create(pool, input).await

6
crates/common/tests/repository_runtime_tests.rs
View File

@@ -79,6 +79,8 @@ impl RuntimeFixture {
"file_extension": ".py"
}
}),
auto_detected: false,
detection_config: json!({}),
}
}
@@ -102,6 +104,8 @@ impl RuntimeFixture {
"file_extension": ".sh"
}
}),
auto_detected: false,
detection_config: json!({}),
}
}
}
@@ -268,6 +272,7 @@ async fn test_update_runtime() {
"method": "npm"
}))),
execution_config: None,
..Default::default()
};
let updated = RuntimeRepository::update(&pool, created.id, update_input.clone())
@@ -299,6 +304,7 @@ async fn test_update_runtime_partial() {
distributions: None,
installation: None,
execution_config: None,
..Default::default()
};
let updated = RuntimeRepository::update(&pool, created.id, update_input.clone())

2
crates/common/tests/repository_worker_tests.rs
View File

@@ -583,6 +583,8 @@ async fn test_worker_with_runtime() {
"file_extension": ".sh"
}
}),
auto_detected: false,
detection_config: json!({}),
};
let runtime = RuntimeRepository::create(&pool, runtime_input)

2
crates/executor/tests/fifo_ordering_integration_test.rs
View File

@@ -81,6 +81,8 @@ async fn _create_test_runtime(pool: &PgPool, suffix: &str) -> i64 {
"file_extension": ".py"
}
}),
auto_detected: false,
detection_config: json!({}),
};
RuntimeRepository::create(pool, runtime_input)

2
crates/executor/tests/policy_enforcer_tests.rs
View File

@@ -76,6 +76,8 @@ async fn create_test_runtime(pool: &PgPool, suffix: &str) -> i64 {
"file_extension": ".py"
}
}),
auto_detected: false,
detection_config: json!({}),
};
let runtime = RuntimeRepository::create(pool, runtime_input)

4
crates/worker/Cargo.toml
View File

@@ -10,6 +10,10 @@ repository.workspace = true
name = "attune-worker"
path = "src/main.rs"
[[bin]]
name = "attune-agent"
path = "src/agent_main.rs"
[dependencies]
attune-common = { path = "../common" }
tokio = { workspace = true }

258
crates/worker/src/agent_main.rs Normal file
View File

@@ -0,0 +1,258 @@
//! Attune Universal Worker Agent
//!
//! This is the entrypoint for the universal worker agent binary (`attune-agent`).
//! Unlike the standard `attune-worker` binary which requires explicit runtime
//! configuration, the agent automatically detects available interpreters in the
//! container environment and configures itself accordingly.
//!
//! ## Usage
//!
//! The agent is designed to be injected into any container image. On startup it:
//!
//! 1. Probes the system for available interpreters (python3, node, bash, etc.)
//! 2. Sets `ATTUNE_WORKER_RUNTIMES` based on what it finds
//! 3. Loads configuration (env vars are the primary config source)
//! 4. Initializes and runs the standard `WorkerService`
//!
//! ## Configuration
//!
//! Environment variables (primary):
//! - `ATTUNE__DATABASE__URL` — PostgreSQL connection string
//! - `ATTUNE__MESSAGE_QUEUE__URL` — RabbitMQ connection string
//! - `ATTUNE_WORKER_RUNTIMES` — Override auto-detection with explicit runtime list
//! - `ATTUNE_CONFIG` — Path to optional config YAML file
//!
//! CLI arguments:
//! - `--config` / `-c` — Path to configuration file (optional)
//! - `--name` / `-n` — Worker name override
//! - `--detect-only` — Run runtime detection, print results, and exit
use anyhow::Result;
use attune_common::config::Config;
use clap::Parser;
use tokio::signal::unix::{signal, SignalKind};
use tracing::{info, warn};
use attune_worker::dynamic_runtime::auto_register_detected_runtimes;
use attune_worker::runtime_detect::{detect_runtimes, print_detection_report};
use attune_worker::service::WorkerService;
#[derive(Parser, Debug)]
#[command(name = "attune-agent")]
#[command(
about = "Attune Universal Worker Agent - Injected into any container to auto-detect and execute actions",
long_about = "The Attune Agent automatically discovers available runtime interpreters \
in the current environment and registers as a worker capable of executing \
actions for those runtimes. It is designed to be injected into arbitrary \
container images without requiring manual runtime configuration."
)]
struct Args {
/// Path to configuration file (optional — env vars are the primary config source)
#[arg(short, long)]
config: Option<String>,
/// Worker name (overrides config and auto-generated name)
#[arg(short, long)]
name: Option<String>,
/// Run runtime detection, print results, and exit without starting the worker
#[arg(long)]
detect_only: bool,
}
#[tokio::main]
async fn main() -> Result<()> {
// Install HMAC-only JWT crypto provider (must be before any token operations)
attune_common::auth::install_crypto_provider();
// Initialize tracing
tracing_subscriber::fmt()
.with_target(false)
.with_thread_ids(true)
.init();
let args = Args::parse();
info!("Starting Attune Universal Worker Agent");
// --- Phase 1: Runtime auto-detection ---
//
// Check if the user has explicitly set ATTUNE_WORKER_RUNTIMES. If so, skip
// auto-detection and respect their override. Otherwise, probe the system for
// available interpreters.
let runtimes_override = std::env::var("ATTUNE_WORKER_RUNTIMES").ok();
// Holds the detected runtimes so we can pass them to WorkerService later.
// Populated only when auto-detection actually runs (no env var override).
let mut agent_detected_runtimes: Option<Vec<attune_worker::runtime_detect::DetectedRuntime>> =
None;
if let Some(ref override_value) = runtimes_override {
info!(
"ATTUNE_WORKER_RUNTIMES already set (override), skipping auto-detection: {}",
override_value
);
} else {
info!("No ATTUNE_WORKER_RUNTIMES override — running auto-detection...");
let detected = detect_runtimes();
if detected.is_empty() {
warn!("No runtimes detected! The agent may not be able to execute any actions.");
} else {
info!("Detected {} runtime(s):", detected.len());
for rt in &detected {
match &rt.version {
Some(ver) => info!(" ✓ {} — {} ({})", rt.name, rt.path, ver),
None => info!(" ✓ {} — {}", rt.name, rt.path),
}
}
// Build comma-separated runtime list and set the env var so that
// Config::load() and WorkerService pick it up downstream.
let runtime_list: Vec<&str> = detected.iter().map(|r| r.name.as_str()).collect();
let runtime_csv = runtime_list.join(",");
info!("Setting ATTUNE_WORKER_RUNTIMES={}", runtime_csv);
std::env::set_var("ATTUNE_WORKER_RUNTIMES", &runtime_csv);
// Stash for Phase 2: pass to WorkerService for rich capability registration
agent_detected_runtimes = Some(detected);
}
}
// --- Handle --detect-only ---
if args.detect_only {
if runtimes_override.is_some() {
// User set an override, but --detect-only should show what's actually
// on this system regardless, so re-run detection.
info!(
"--detect-only: re-running detection to show what is available on this system..."
);
println!("NOTE: ATTUNE_WORKER_RUNTIMES is set — auto-detection was skipped during normal startup.");
println!(" Showing what auto-detection would find on this system:");
println!();
let detected = detect_runtimes();
print_detection_report(&detected);
} else {
// We already ran detection above; re-run to get a fresh Vec for the report
// (the previous one was consumed by env var setup).
let detected = detect_runtimes();
print_detection_report(&detected);
}
return Ok(());
}
// --- Phase 2: Load configuration ---
if let Some(config_path) = args.config {
std::env::set_var("ATTUNE_CONFIG", config_path);
}
let mut config = Config::load()?;
config.validate()?;
// Override worker name if provided via CLI
if let Some(name) = args.name {
if let Some(ref mut worker_config) = config.worker {
worker_config.name = Some(name);
} else {
config.worker = Some(attune_common::config::WorkerConfig {
name: Some(name),
worker_type: None,
runtime_id: None,
host: None,
port: None,
capabilities: None,
max_concurrent_tasks: 10,
heartbeat_interval: 30,
task_timeout: 300,
max_stdout_bytes: 10 * 1024 * 1024,
max_stderr_bytes: 10 * 1024 * 1024,
shutdown_timeout: Some(30),
stream_logs: true,
});
}
}
info!("Configuration loaded successfully");
info!("Environment: {}", config.environment);
// --- Phase 2b: Dynamic runtime registration ---
//
// Before creating the WorkerService (which loads runtimes from the DB into
// its runtime registry), ensure that every detected runtime has a
// corresponding entry in the database. This handles the case where the
// agent detects a runtime (e.g., Ruby) that has a template in the core
// pack but hasn't been explicitly loaded by this agent before.
if let Some(ref detected) = agent_detected_runtimes {
info!(
"Ensuring {} detected runtime(s) are registered in the database...",
detected.len()
);
// We need a temporary DB connection for dynamic registration.
// WorkerService::new() will create its own connection, so this is
// a short-lived pool just for the registration step.
let db = attune_common::db::Database::new(&config.database).await?;
let pool = db.pool().clone();
match auto_register_detected_runtimes(&pool, detected).await {
Ok(count) => {
if count > 0 {
info!(
"Dynamic registration complete: {} new runtime(s) added to database",
count
);
} else {
info!("Dynamic registration: all detected runtimes already in database");
}
}
Err(e) => {
warn!(
"Dynamic runtime registration failed (non-fatal, continuing): {}",
e
);
}
}
}
// --- Phase 3: Initialize and run the worker service ---
let service = WorkerService::new(config).await?;
// If we auto-detected runtimes, pass them to the worker service so that
// registration includes the full `detected_interpreters` capability
// (binary paths + versions) and the `agent_mode` flag.
let mut service = if let Some(detected) = agent_detected_runtimes {
info!(
"Passing {} detected runtime(s) to worker registration",
detected.len()
);
service.with_detected_runtimes(detected)
} else {
service
};
info!("Attune Agent is ready");
service.start().await?;
// Setup signal handlers for graceful shutdown
let mut sigint = signal(SignalKind::interrupt())?;
let mut sigterm = signal(SignalKind::terminate())?;
tokio::select! {
_ = sigint.recv() => {
info!("Received SIGINT signal");
}
_ = sigterm.recv() => {
info!("Received SIGTERM signal");
}
}
info!("Shutting down gracefully...");
service.stop().await?;
info!("Attune Agent shutdown complete");
Ok(())
}

518
crates/worker/src/dynamic_runtime.rs Normal file
View File

@@ -0,0 +1,518 @@
//! Dynamic Runtime Registration Module
//!
//! When the agent detects an interpreter on the local system (e.g., Ruby, Go, Perl)
//! that does not yet have a corresponding runtime entry in the database, this module
//! handles auto-registering it so that the normal runtime-loading pipeline in
//! `WorkerService::new()` picks it up.
//!
//! ## Registration Strategy
//!
//! For each detected runtime the agent found:
//!
//! 1. **Look up by name** in the database using alias-aware matching
//! (via [`normalize_runtime_name`]).
//! 2. **If found** → already registered (either from a pack YAML or a previous
//! agent run). Nothing to do.
//! 3. **If not found** → search for a runtime *template* in loaded packs whose
//! normalized name matches. Templates are pack-registered runtimes (e.g.,
//! `core.ruby`) that provide the full `execution_config` needed to invoke
//! the interpreter, manage environments, and install dependencies.
//! 4. **If a template is found** → clone it as an auto-detected runtime with
//! `auto_detected = true` and populate `detection_config` with what the
//! agent discovered (binary path, version, etc.).
//! 5. **If no template exists** → create a minimal runtime with just the
//! detected interpreter binary path and file extension. This lets the agent
//! execute simple scripts immediately, even without a full template.
//! 6. Mark all auto-registered runtimes with `auto_detected = true`.
use attune_common::error::Result;
use attune_common::models::runtime::Runtime;
use attune_common::repositories::runtime::{CreateRuntimeInput, RuntimeRepository};
use attune_common::repositories::{Create, FindByRef, List};
use attune_common::runtime_detection::normalize_runtime_name;
use serde_json::json;
use sqlx::PgPool;
use tracing::{debug, info, warn};
use crate::runtime_detect::DetectedRuntime;
/// Canonical file extensions for runtimes that the auto-detection module knows
/// about. Used when creating minimal runtime entries without a template.
fn default_file_extension(runtime_name: &str) -> Option<&'static str> {
match runtime_name {
"shell" => Some(".sh"),
"python" => Some(".py"),
"node" => Some(".js"),
"ruby" => Some(".rb"),
"go" => Some(".go"),
"java" => Some(".java"),
"perl" => Some(".pl"),
"r" => Some(".R"),
_ => None,
}
}
/// Ensure that every detected runtime has a corresponding entry in the
/// `runtime` table. Runtimes that already exist (from pack loading or a
/// previous agent run) are left untouched. Missing runtimes are created
/// either from a matching pack template or as a minimal auto-detected entry.
///
/// This function should be called **before** `WorkerService::new()` so that
/// the normal runtime-loading pipeline finds all detected runtimes in the DB.
///
/// Returns the number of runtimes that were newly registered.
pub async fn auto_register_detected_runtimes(
pool: &PgPool,
detected: &[DetectedRuntime],
) -> Result<usize> {
if detected.is_empty() {
return Ok(0);
}
info!(
"Checking {} detected runtime(s) for dynamic registration...",
detected.len()
);
// Load all existing runtimes once to avoid repeated queries.
let existing_runtimes = RuntimeRepository::list(pool).await.unwrap_or_default();
let mut registered_count = 0;
for detected_rt in detected {
let canonical_name = normalize_runtime_name(&detected_rt.name);
// Check if a runtime with a matching name already exists in the DB.
// We normalize both sides for alias-aware comparison.
let already_exists = existing_runtimes.iter().any(|r| {
let db_name = r.name.to_ascii_lowercase();
normalize_runtime_name(&db_name) == canonical_name
});
if already_exists {
debug!(
"Runtime '{}' (canonical: '{}') already exists in database, skipping",
detected_rt.name, canonical_name
);
continue;
}
// No existing runtime — try to find a template from loaded packs.
// Templates are pack-registered runtimes whose normalized name matches
// (e.g., `core.ruby` for detected runtime "ruby"). Since we already
// checked `existing_runtimes` above and found nothing, we look for
// runtimes by ref pattern. Common convention: `core.<name>`.
let template_ref = format!("core.{}", canonical_name);
let template = RuntimeRepository::find_by_ref(pool, &template_ref)
.await
.unwrap_or(None);
let detection_config = build_detection_config(detected_rt);
if let Some(tmpl) = template {
// Clone the template as an auto-detected runtime.
// The template already has the full execution_config, distributions, etc.
// We just re-create it with auto_detected = true.
info!(
"Found template '{}' for detected runtime '{}', registering auto-detected clone",
tmpl.r#ref, detected_rt.name
);
// Use a distinct ref so we don't collide with the template.
let auto_ref = format!("auto.{}", canonical_name);
// Check if the auto ref already exists (race condition / previous run)
if RuntimeRepository::find_by_ref(pool, &auto_ref)
.await
.unwrap_or(None)
.is_some()
{
debug!(
"Auto-detected runtime '{}' already registered from a previous run",
auto_ref
);
continue;
}
let input = CreateRuntimeInput {
r#ref: auto_ref.clone(),
pack: tmpl.pack,
pack_ref: tmpl.pack_ref.clone(),
description: Some(format!(
"Auto-detected {} runtime (from template {})",
detected_rt.name, tmpl.r#ref
)),
name: tmpl.name.clone(),
distributions: tmpl.distributions.clone(),
installation: tmpl.installation.clone(),
execution_config: build_execution_config_from_template(&tmpl, detected_rt),
auto_detected: true,
detection_config,
};
match RuntimeRepository::create(pool, input).await {
Ok(rt) => {
info!(
"Auto-registered runtime '{}' (ID: {}) from template '{}'",
auto_ref, rt.id, tmpl.r#ref
);
registered_count += 1;
}
Err(e) => {
// Unique constraint violation is fine (concurrent agent start)
warn!("Failed to auto-register runtime '{}': {}", auto_ref, e);
}
}
} else {
// No template found — create a minimal runtime entry.
info!(
"No template found for detected runtime '{}', creating minimal entry",
detected_rt.name
);
let auto_ref = format!("auto.{}", canonical_name);
if RuntimeRepository::find_by_ref(pool, &auto_ref)
.await
.unwrap_or(None)
.is_some()
{
debug!(
"Auto-detected runtime '{}' already registered from a previous run",
auto_ref
);
continue;
}
let execution_config = build_minimal_execution_config(detected_rt);
let input = CreateRuntimeInput {
r#ref: auto_ref.clone(),
pack: None,
pack_ref: None,
description: Some(format!(
"Auto-detected {} runtime at {}",
detected_rt.name, detected_rt.path
)),
name: capitalize_runtime_name(canonical_name),
distributions: build_minimal_distributions(detected_rt),
installation: None,
execution_config,
auto_detected: true,
detection_config,
};
match RuntimeRepository::create(pool, input).await {
Ok(rt) => {
info!(
"Auto-registered minimal runtime '{}' (ID: {})",
auto_ref, rt.id
);
registered_count += 1;
}
Err(e) => {
warn!("Failed to auto-register runtime '{}': {}", auto_ref, e);
}
}
}
}
if registered_count > 0 {
info!(
"Dynamic runtime registration complete: {} new runtime(s) registered",
registered_count
);
} else {
info!("Dynamic runtime registration complete: all detected runtimes already in database");
}
Ok(registered_count)
}
/// Build the `detection_config` JSONB value from a detected runtime.
/// This metadata records how the agent discovered this runtime, enabling
/// re-verification and diagnostics.
fn build_detection_config(detected: &DetectedRuntime) -> serde_json::Value {
let mut config = json!({
"detected_path": detected.path,
"detected_name": detected.name,
});
if let Some(ref version) = detected.version {
config["detected_version"] = json!(version);
}
config
}
/// Build an execution config based on a template runtime, but with the
/// detected interpreter path substituted in. This ensures the auto-detected
/// runtime uses the actual binary path found on the system.
fn build_execution_config_from_template(
template: &Runtime,
detected: &DetectedRuntime,
) -> serde_json::Value {
let mut config = template.execution_config.clone();
// If the template has an interpreter config, update the binary path
// to the one we actually detected on this system.
if let Some(interpreter) = config.get_mut("interpreter") {
if let Some(obj) = interpreter.as_object_mut() {
obj.insert("binary".to_string(), json!(detected.path));
}
}
// If the template has an environment config with an interpreter_path
// that uses a template variable, leave it as-is (it will be resolved
// at execution time). But if it's a hardcoded absolute path, update it.
if let Some(env) = config.get_mut("environment") {
if let Some(obj) = env.as_object_mut() {
if let Some(interp_path) = obj.get("interpreter_path") {
if let Some(path_str) = interp_path.as_str() {
// Only leave template variables alone
if !path_str.contains('{') {
obj.insert("interpreter_path".to_string(), json!(detected.path));
}
}
}
}
}
config
}
/// Build a minimal execution config for a runtime with no template.
/// This provides enough information for `ProcessRuntime` to invoke the
/// interpreter directly, without environment or dependency management.
fn build_minimal_execution_config(detected: &DetectedRuntime) -> serde_json::Value {
let canonical = normalize_runtime_name(&detected.name);
let file_ext = default_file_extension(canonical);
let mut config = json!({
"interpreter": {
"binary": detected.path,
"args": [],
}
});
if let Some(ext) = file_ext {
config["interpreter"]["file_extension"] = json!(ext);
}
config
}
/// Build minimal distributions metadata for a runtime with no template.
/// Includes a basic verification command using the detected binary path.
fn build_minimal_distributions(detected: &DetectedRuntime) -> serde_json::Value {
json!({
"verification": {
"commands": [
{
"binary": &detected.path,
"args": ["--version"],
"exit_code": 0,
"priority": 1
}
]
}
})
}
/// Capitalize a runtime name for display (e.g., "ruby" → "Ruby", "r" → "R").
fn capitalize_runtime_name(name: &str) -> String {
let mut chars = name.chars();
match chars.next() {
None => String::new(),
Some(first) => {
let upper: String = first.to_uppercase().collect();
format!("{}{}", upper, chars.as_str())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_file_extension() {
assert_eq!(default_file_extension("shell"), Some(".sh"));
assert_eq!(default_file_extension("python"), Some(".py"));
assert_eq!(default_file_extension("node"), Some(".js"));
assert_eq!(default_file_extension("ruby"), Some(".rb"));
assert_eq!(default_file_extension("go"), Some(".go"));
assert_eq!(default_file_extension("java"), Some(".java"));
assert_eq!(default_file_extension("perl"), Some(".pl"));
assert_eq!(default_file_extension("r"), Some(".R"));
assert_eq!(default_file_extension("unknown"), None);
}
#[test]
fn test_capitalize_runtime_name() {
assert_eq!(capitalize_runtime_name("ruby"), "Ruby");
assert_eq!(capitalize_runtime_name("go"), "Go");
assert_eq!(capitalize_runtime_name("r"), "R");
assert_eq!(capitalize_runtime_name("perl"), "Perl");
assert_eq!(capitalize_runtime_name("java"), "Java");
assert_eq!(capitalize_runtime_name(""), "");
}
#[test]
fn test_build_detection_config_with_version() {
let detected = DetectedRuntime {
name: "ruby".to_string(),
path: "/usr/bin/ruby".to_string(),
version: Some("3.3.0".to_string()),
};
let config = build_detection_config(&detected);
assert_eq!(config["detected_path"], "/usr/bin/ruby");
assert_eq!(config["detected_name"], "ruby");
assert_eq!(config["detected_version"], "3.3.0");
}
#[test]
fn test_build_detection_config_without_version() {
let detected = DetectedRuntime {
name: "perl".to_string(),
path: "/usr/bin/perl".to_string(),
version: None,
};
let config = build_detection_config(&detected);
assert_eq!(config["detected_path"], "/usr/bin/perl");
assert_eq!(config["detected_name"], "perl");
assert!(config.get("detected_version").is_none());
}
#[test]
fn test_build_minimal_execution_config() {
let detected = DetectedRuntime {
name: "ruby".to_string(),
path: "/usr/bin/ruby".to_string(),
version: Some("3.3.0".to_string()),
};
let config = build_minimal_execution_config(&detected);
assert_eq!(config["interpreter"]["binary"], "/usr/bin/ruby");
assert_eq!(config["interpreter"]["file_extension"], ".rb");
assert_eq!(config["interpreter"]["args"], json!([]));
}
#[test]
fn test_build_minimal_execution_config_unknown_runtime() {
let detected = DetectedRuntime {
name: "custom".to_string(),
path: "/opt/custom/bin/custom".to_string(),
version: None,
};
let config = build_minimal_execution_config(&detected);
assert_eq!(config["interpreter"]["binary"], "/opt/custom/bin/custom");
// Unknown runtime has no file extension
assert!(config["interpreter"].get("file_extension").is_none());
}
#[test]
fn test_build_minimal_distributions() {
let detected = DetectedRuntime {
name: "ruby".to_string(),
path: "/usr/bin/ruby".to_string(),
version: Some("3.3.0".to_string()),
};
let distros = build_minimal_distributions(&detected);
let commands = distros["verification"]["commands"].as_array().unwrap();
assert_eq!(commands.len(), 1);
assert_eq!(commands[0]["binary"], "/usr/bin/ruby");
}
#[test]
fn test_build_execution_config_from_template_updates_binary() {
let template = Runtime {
id: 1,
r#ref: "core.ruby".to_string(),
pack: Some(1),
pack_ref: Some("core".to_string()),
description: Some("Ruby Runtime".to_string()),
name: "Ruby".to_string(),
distributions: json!({}),
installation: None,
installers: json!({}),
execution_config: json!({
"interpreter": {
"binary": "ruby",
"args": [],
"file_extension": ".rb"
},
"env_vars": {
"GEM_HOME": "{env_dir}/gems"
}
}),
auto_detected: false,
detection_config: json!({}),
created: chrono::Utc::now(),
updated: chrono::Utc::now(),
};
let detected = DetectedRuntime {
name: "ruby".to_string(),
path: "/usr/local/bin/ruby3.3".to_string(),
version: Some("3.3.0".to_string()),
};
let config = build_execution_config_from_template(&template, &detected);
// Binary should be updated to the detected path
assert_eq!(config["interpreter"]["binary"], "/usr/local/bin/ruby3.3");
// Other fields should be preserved from the template
assert_eq!(config["interpreter"]["file_extension"], ".rb");
assert_eq!(config["env_vars"]["GEM_HOME"], "{env_dir}/gems");
}
#[test]
fn test_build_execution_config_from_template_preserves_template_vars() {
let template = Runtime {
id: 1,
r#ref: "core.python".to_string(),
pack: Some(1),
pack_ref: Some("core".to_string()),
description: None,
name: "Python".to_string(),
distributions: json!({}),
installation: None,
installers: json!({}),
execution_config: json!({
"interpreter": {
"binary": "python3",
"file_extension": ".py"
},
"environment": {
"interpreter_path": "{env_dir}/bin/python3",
"create_command": ["python3", "-m", "venv", "{env_dir}"]
}
}),
auto_detected: false,
detection_config: json!({}),
created: chrono::Utc::now(),
updated: chrono::Utc::now(),
};
let detected = DetectedRuntime {
name: "python".to_string(),
path: "/usr/bin/python3.12".to_string(),
version: Some("3.12.1".to_string()),
};
let config = build_execution_config_from_template(&template, &detected);
// Binary should be updated
assert_eq!(config["interpreter"]["binary"], "/usr/bin/python3.12");
// Template-variable interpreter_path should be preserved (contains '{')
assert_eq!(
config["environment"]["interpreter_path"],
"{env_dir}/bin/python3"
);
}
}

1
crates/worker/src/executor.rs
View File

@@ -413,6 +413,7 @@ impl ActionExecutor {
match sqlx::query_as::<_, RuntimeModel>(
r#"SELECT id, ref, pack, pack_ref, description, name,
distributions, installation, installers, execution_config,
auto_detected, detection_config,
created, updated
FROM runtime WHERE id = $1"#,
)

6
crates/worker/src/lib.rs
View File

@@ -4,16 +4,19 @@
//! which executes actions in various runtime environments.
pub mod artifacts;
pub mod dynamic_runtime;
pub mod env_setup;
pub mod executor;
pub mod heartbeat;
pub mod registration;
pub mod runtime;
pub mod runtime_detect;
pub mod secrets;
pub mod service;
pub mod version_verify;
// Re-export commonly used types
pub use dynamic_runtime::auto_register_detected_runtimes;
pub use executor::ActionExecutor;
pub use heartbeat::HeartbeatManager;
pub use registration::WorkerRegistration;
@@ -21,7 +24,8 @@ pub use runtime::{
ExecutionContext, ExecutionResult, LocalRuntime, NativeRuntime, ProcessRuntime, Runtime,
RuntimeError, RuntimeResult,
};
pub use runtime_detect::DetectedRuntime;
pub use secrets::SecretManager;
pub use service::WorkerService;
pub use service::{StartupMode, WorkerService};
// Re-export test executor from common (shared business logic)
pub use attune_common::test_executor::{TestConfig, TestExecutor};

47
crates/worker/src/registration.rs
View File

@@ -13,6 +13,8 @@ use sqlx::PgPool;
use std::collections::HashMap;
use tracing::{info, warn};
use crate::runtime_detect::DetectedRuntime;
/// Worker registration manager
pub struct WorkerRegistration {
pool: PgPool,
@@ -100,6 +102,51 @@ impl WorkerRegistration {
}
}
/// Store detected runtime interpreter metadata in capabilities.
///
/// This is used by the agent (`attune-agent`) to record the full details of
/// auto-detected interpreters — binary paths and versions — alongside the
/// simple `runtimes` string list used for backward compatibility.
///
/// The data is stored under the `detected_interpreters` capability key as a
/// JSON array of objects:
/// ```json
/// [
/// {"name": "python", "path": "/usr/bin/python3", "version": "3.12.1"},
/// {"name": "shell", "path": "/bin/bash", "version": "5.2.15"}
/// ]
/// ```
pub fn set_detected_runtimes(&mut self, runtimes: Vec<DetectedRuntime>) {
let interpreters: Vec<serde_json::Value> = runtimes
.iter()
.map(|rt| {
json!({
"name": rt.name,
"path": rt.path,
"version": rt.version,
})
})
.collect();
self.capabilities
.insert("detected_interpreters".to_string(), json!(interpreters));
info!(
"Stored {} detected interpreter(s) in capabilities",
runtimes.len()
);
}
/// Mark this worker as running in agent mode.
///
/// Agent-mode workers auto-detect their runtimes at startup (as opposed to
/// being configured via `ATTUNE_WORKER_RUNTIMES` or config files). Setting
/// this flag allows the system to distinguish agents from standard workers.
pub fn set_agent_mode(&mut self, is_agent: bool) {
self.capabilities
.insert("agent_mode".to_string(), json!(is_agent));
}
/// Detect available runtimes using the unified runtime detector
pub async fn detect_capabilities(&mut self, config: &Config) -> Result<()> {
info!("Detecting worker capabilities...");

42
crates/worker/src/runtime/process.rs
View File

@@ -615,18 +615,46 @@ impl Runtime for ProcessRuntime {
None
};
// Runtime environments are set up proactively — either at worker startup
// (scanning all registered packs) or via pack.registered MQ events when a
// new pack is installed. We only log a warning here if the expected
// environment directory is missing so operators can investigate.
// Lazy environment setup: if the environment directory doesn't exist but
// should (i.e., there's an environment config and the pack dir exists),
// create it on-demand. This is the primary code path for agent mode where
// proactive startup setup is skipped, but it also serves as a safety net
// for standard workers if the environment was somehow missed.
if effective_config.environment.is_some() && pack_dir.exists() && !env_dir.exists() {
warn!(
info!(
"Runtime environment for pack '{}' not found at {}. \
The environment should have been created at startup or on pack registration. \
Proceeding with system interpreter as fallback.",
Creating on first use (lazy setup).",
context.action_ref,
env_dir.display(),
);
let setup_runtime = ProcessRuntime::new(
self.runtime_name.clone(),
effective_config.clone(),
self.packs_base_dir.clone(),
self.runtime_envs_dir.clone(),
);
match setup_runtime
.setup_pack_environment(&pack_dir, &env_dir)
.await
{
Ok(()) => {
info!(
"Successfully created environment for pack '{}' at {} (lazy setup)",
context.action_ref,
env_dir.display(),
);
}
Err(e) => {
warn!(
"Failed to create environment for pack '{}' at {}: {}. \
Proceeding with system interpreter as fallback.",
context.action_ref,
env_dir.display(),
e,
);
}
}
}
// If the environment directory exists but contains a broken interpreter

544
crates/worker/src/runtime_detect.rs Normal file
View File

@@ -0,0 +1,544 @@
//! Runtime Auto-Detection Module
//!
//! Provides lightweight, database-free runtime detection for the Universal Worker Agent.
//! Unlike [`attune_common::runtime_detection::RuntimeDetector`] which queries the database
//! for runtime definitions and verification metadata, this module probes the local system
//! directly by checking for well-known interpreter binaries on PATH.
//!
//! This is designed for the agent entrypoint (`attune-agent`) which is injected into
//! arbitrary containers and must discover what runtimes are available without any
//! database connectivity at detection time.
//!
//! # Detection Strategy
//!
//! For each candidate runtime, the detector:
//! 1. Checks if a binary exists and is executable using `which`-style PATH lookup
//! 2. Optionally runs a version command (e.g., `python3 --version`) to capture the version
//! 3. Returns a list of [`DetectedRuntime`] structs with name, path, and version info
//!
//! # Supported Runtimes
//!
//! | Runtime | Binaries checked (in order) | Version command |
//! |----------|-------------------------------|-------------------------|
//! | shell | `bash`, `sh` | `bash --version` |
//! | python | `python3`, `python` | `python3 --version` |
//! | node | `node`, `nodejs` | `node --version` |
//! | ruby | `ruby` | `ruby --version` |
//! | go | `go` | `go version` |
//! | java | `java` | `java -version` |
//! | r | `Rscript` | `Rscript --version` |
//! | perl | `perl` | `perl --version` |
use serde::{Deserialize, Serialize};
use std::fmt;
use std::process::Command;
use tracing::{debug, info};
/// A runtime interpreter discovered on the local system.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DetectedRuntime {
/// Canonical runtime name (e.g., "shell", "python", "node").
/// These names align with the normalized names from
/// [`attune_common::runtime_detection::normalize_runtime_name`].
pub name: String,
/// Absolute path to the interpreter binary (as resolved by `which`).
pub path: String,
/// Version string if a version check command succeeded (e.g., "3.12.1").
pub version: Option<String>,
}
impl fmt::Display for DetectedRuntime {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.version {
Some(v) => write!(f, "{} ({}, v{})", self.name, self.path, v),
None => write!(f, "{} ({})", self.name, self.path),
}
}
}
/// A candidate runtime to probe for during detection.
struct RuntimeCandidate {
/// Canonical name for this runtime (used in ATTUNE_WORKER_RUNTIMES).
name: &'static str,
/// Binary names to try, in priority order. The first one found wins.
binaries: &'static [&'static str],
/// Arguments to pass to the binary to get a version string.
version_args: &'static [&'static str],
/// How to extract the version from command output.
version_parser: VersionParser,
}
/// Strategy for parsing version output from a command.
enum VersionParser {
/// Extract a version pattern like "X.Y.Z" from the combined stdout+stderr output.
/// This handles the common case where the version appears somewhere in the output
/// (e.g., "Python 3.12.1", "node v20.11.0", "go1.22.0").
SemverLike,
/// Java uses `-version` which writes to stderr, and the format is
/// `openjdk version "21.0.1"` or `java version "1.8.0_392"`.
JavaStyle,
}
/// All candidate runtimes to probe, in detection order.
fn candidates() -> Vec<RuntimeCandidate> {
vec![
RuntimeCandidate {
name: "shell",
binaries: &["bash", "sh"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "python",
binaries: &["python3", "python"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "node",
binaries: &["node", "nodejs"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "ruby",
binaries: &["ruby"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "go",
binaries: &["go"],
version_args: &["version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "java",
binaries: &["java"],
version_args: &["-version"],
version_parser: VersionParser::JavaStyle,
},
RuntimeCandidate {
name: "r",
binaries: &["Rscript"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
RuntimeCandidate {
name: "perl",
binaries: &["perl"],
version_args: &["--version"],
version_parser: VersionParser::SemverLike,
},
]
}
/// Detect available runtimes by probing the local system for known interpreter binaries.
///
/// This function performs synchronous subprocess calls (`std::process::Command`) since
/// it is a one-time startup operation. It checks each candidate runtime's binaries
/// in priority order using `which`-style PATH lookup, and optionally captures the
/// interpreter version.
///
/// # Returns
///
/// A vector of [`DetectedRuntime`] for each runtime that was found on the system.
/// The order matches the detection order (shell first, then python, node, etc.).
///
/// # Example
///
/// ```no_run
/// use attune_worker::runtime_detect::detect_runtimes;
///
/// let runtimes = detect_runtimes();
/// for rt in &runtimes {
/// println!("Found: {}", rt);
/// }
/// // Convert to ATTUNE_WORKER_RUNTIMES format
/// let names: Vec<&str> = runtimes.iter().map(|r| r.name.as_str()).collect();
/// println!("ATTUNE_WORKER_RUNTIMES={}", names.join(","));
/// ```
pub fn detect_runtimes() -> Vec<DetectedRuntime> {
info!("Starting runtime auto-detection...");
let mut detected = Vec::new();
for candidate in candidates() {
match detect_single_runtime(&candidate) {
Some(runtime) => {
info!(" ✓ Detected: {}", runtime);
detected.push(runtime);
}
None => {
debug!(" ✗ Not found: {}", candidate.name);
}
}
}
info!(
"Runtime auto-detection complete: found {} runtime(s): [{}]",
detected.len(),
detected
.iter()
.map(|r| r.name.as_str())
.collect::<Vec<_>>()
.join(", ")
);
detected
}
/// Attempt to detect a single runtime by checking its candidate binaries.
fn detect_single_runtime(candidate: &RuntimeCandidate) -> Option<DetectedRuntime> {
for binary in candidate.binaries {
if let Some(path) = which_binary(binary) {
debug!(
"Found {} at {} (for runtime '{}')",
binary, path, candidate.name
);
// Attempt to get version info (non-fatal if it fails)
let version = get_version(&path, candidate.version_args, &candidate.version_parser);
return Some(DetectedRuntime {
name: candidate.name.to_string(),
path,
version,
});
}
}
None
}
/// Look up a binary on PATH, similar to the `which` command.
///
/// Uses `which <binary>` on the system to resolve the full path.
/// Returns `None` if the binary is not found or `which` fails.
fn which_binary(binary: &str) -> Option<String> {
// First check well-known absolute paths for shell interpreters
// (these may not be on PATH in minimal containers)
if binary == "bash" || binary == "sh" {
let absolute_path = format!("/bin/{}", binary);
if std::path::Path::new(&absolute_path).exists() {
return Some(absolute_path);
}
}
// Fall back to PATH lookup via `which`
match Command::new("which").arg(binary).output() {
Ok(output) => {
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !path.is_empty() {
Some(path)
} else {
None
}
} else {
None
}
}
Err(e) => {
// `which` itself not found — try `command -v` as fallback
debug!("'which' command failed ({}), trying 'command -v'", e);
match Command::new("sh")
.args(["-c", &format!("command -v {}", binary)])
.output()
{
Ok(output) if output.status.success() => {
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !path.is_empty() {
Some(path)
} else {
None
}
}
_ => None,
}
}
}
}
/// Run a version command and parse the version string from the output.
fn get_version(binary_path: &str, version_args: &[&str], parser: &VersionParser) -> Option<String> {
let output = match Command::new(binary_path).args(version_args).output() {
Ok(output) => output,
Err(e) => {
debug!("Failed to run version command for {}: {}", binary_path, e);
return None;
}
};
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
let combined = format!("{}{}", stdout, stderr);
match parser {
VersionParser::SemverLike => parse_semver_like(&combined),
VersionParser::JavaStyle => parse_java_version(&combined),
}
}
/// Extract a semver-like version (X.Y.Z or X.Y) from output text.
///
/// Handles common patterns:
/// - "Python 3.12.1"
/// - "node v20.11.0"
/// - "go version go1.22.0 linux/amd64"
/// - "GNU bash, version 5.2.15(1)-release"
/// - "ruby 3.2.2 (2023-03-30 revision e51014f9c0)"
/// - "perl 5, version 36, subversion 0 (v5.36.0)"
fn parse_semver_like(output: &str) -> Option<String> {
// Try to find a pattern like X.Y.Z or X.Y (with optional leading 'v')
// Also handle go's "go1.22.0" format
let re = regex::Regex::new(r"(?:v|go)?(\d+\.\d+(?:\.\d+)?)").ok()?;
if let Some(captures) = re.captures(output) {
captures.get(1).map(|m| m.as_str().to_string())
} else {
None
}
}
/// Parse Java's peculiar version output format.
///
/// Java writes to stderr and uses formats like:
/// - `openjdk version "21.0.1" 2023-10-17`
/// - `java version "1.8.0_392"`
fn parse_java_version(output: &str) -> Option<String> {
// Look for version inside quotes first
let quoted_re = regex::Regex::new(r#"version\s+"([^"]+)""#).ok()?;
if let Some(captures) = quoted_re.captures(output) {
return captures.get(1).map(|m| m.as_str().to_string());
}
// Fall back to semver-like parsing
parse_semver_like(output)
}
/// Format detected runtimes as a comma-separated string suitable for
/// the `ATTUNE_WORKER_RUNTIMES` environment variable.
///
/// # Example
///
/// ```no_run
/// use attune_worker::runtime_detect::{detect_runtimes, format_as_env_value};
///
/// let runtimes = detect_runtimes();
/// let env_val = format_as_env_value(&runtimes);
/// // e.g., "shell,python,node"
/// ```
pub fn format_as_env_value(runtimes: &[DetectedRuntime]) -> String {
runtimes
.iter()
.map(|r| r.name.as_str())
.collect::<Vec<_>>()
.join(",")
}
/// Print a human-readable detection report to stdout.
///
/// Used by the `--detect-only` flag to show detection results and exit.
pub fn print_detection_report(runtimes: &[DetectedRuntime]) {
println!("=== Attune Agent Runtime Detection Report ===");
println!();
if runtimes.is_empty() {
println!("No runtimes detected!");
println!();
println!("The agent could not find any supported interpreter binaries.");
println!("Ensure at least one of the following is installed and on PATH:");
println!(" - bash / sh (shell scripts)");
println!(" - python3 / python (Python scripts)");
println!(" - node / nodejs (Node.js scripts)");
println!(" - ruby (Ruby scripts)");
println!(" - go (Go programs)");
println!(" - java (Java programs)");
println!(" - Rscript (R scripts)");
println!(" - perl (Perl scripts)");
} else {
println!("Detected {} runtime(s):", runtimes.len());
println!();
for rt in runtimes {
let version_str = rt.version.as_deref().unwrap_or("unknown version");
println!(" ✓ {:<10} {} ({})", rt.name, rt.path, version_str);
}
}
println!();
println!("ATTUNE_WORKER_RUNTIMES={}", format_as_env_value(runtimes));
println!();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_semver_like_python() {
assert_eq!(
parse_semver_like("Python 3.12.1"),
Some("3.12.1".to_string())
);
}
#[test]
fn test_parse_semver_like_node() {
assert_eq!(parse_semver_like("v20.11.0"), Some("20.11.0".to_string()));
}
#[test]
fn test_parse_semver_like_go() {
assert_eq!(
parse_semver_like("go version go1.22.0 linux/amd64"),
Some("1.22.0".to_string())
);
}
#[test]
fn test_parse_semver_like_bash() {
assert_eq!(
parse_semver_like("GNU bash, version 5.2.15(1)-release (x86_64-pc-linux-gnu)"),
Some("5.2.15".to_string())
);
}
#[test]
fn test_parse_semver_like_ruby() {
assert_eq!(
parse_semver_like("ruby 3.2.2 (2023-03-30 revision e51014f9c0) [x86_64-linux]"),
Some("3.2.2".to_string())
);
}
#[test]
fn test_parse_semver_like_two_part() {
assert_eq!(
parse_semver_like("SomeRuntime 1.5"),
Some("1.5".to_string())
);
}
#[test]
fn test_parse_semver_like_no_match() {
assert_eq!(parse_semver_like("no version here"), None);
}
#[test]
fn test_parse_java_version_openjdk() {
assert_eq!(
parse_java_version(r#"openjdk version "21.0.1" 2023-10-17"#),
Some("21.0.1".to_string())
);
}
#[test]
fn test_parse_java_version_legacy() {
assert_eq!(
parse_java_version(r#"java version "1.8.0_392""#),
Some("1.8.0_392".to_string())
);
}
#[test]
fn test_format_as_env_value_empty() {
let runtimes: Vec<DetectedRuntime> = vec![];
assert_eq!(format_as_env_value(&runtimes), "");
}
#[test]
fn test_format_as_env_value_multiple() {
let runtimes = vec![
DetectedRuntime {
name: "shell".to_string(),
path: "/bin/bash".to_string(),
version: Some("5.2.15".to_string()),
},
DetectedRuntime {
name: "python".to_string(),
path: "/usr/bin/python3".to_string(),
version: Some("3.12.1".to_string()),
},
DetectedRuntime {
name: "node".to_string(),
path: "/usr/bin/node".to_string(),
version: None,
},
];
assert_eq!(format_as_env_value(&runtimes), "shell,python,node");
}
#[test]
fn test_detected_runtime_display_with_version() {
let rt = DetectedRuntime {
name: "python".to_string(),
path: "/usr/bin/python3".to_string(),
version: Some("3.12.1".to_string()),
};
assert_eq!(format!("{}", rt), "python (/usr/bin/python3, v3.12.1)");
}
#[test]
fn test_detected_runtime_display_without_version() {
let rt = DetectedRuntime {
name: "shell".to_string(),
path: "/bin/bash".to_string(),
version: None,
};
assert_eq!(format!("{}", rt), "shell (/bin/bash)");
}
#[test]
fn test_detect_runtimes_runs_without_panic() {
// This test verifies the detection logic doesn't panic,
// regardless of what's actually installed on the system.
let runtimes = detect_runtimes();
// We should at least find a shell on any Unix system
// but we don't assert that since test environments vary.
let _ = runtimes;
}
#[test]
fn test_which_binary_sh() {
// /bin/sh should exist on virtually all Unix systems
let result = which_binary("sh");
assert!(result.is_some(), "Expected to find 'sh' on this system");
}
#[test]
fn test_which_binary_nonexistent() {
let result = which_binary("definitely_not_a_real_binary_xyz123");
assert!(result.is_none());
}
#[test]
fn test_candidates_order() {
let c = candidates();
assert_eq!(c[0].name, "shell");
assert_eq!(c[1].name, "python");
assert_eq!(c[2].name, "node");
assert_eq!(c[3].name, "ruby");
assert_eq!(c[4].name, "go");
assert_eq!(c[5].name, "java");
assert_eq!(c[6].name, "r");
assert_eq!(c[7].name, "perl");
}
#[test]
fn test_candidates_binaries_priority() {
let c = candidates();
// shell prefers bash over sh
assert_eq!(c[0].binaries, &["bash", "sh"]);
// python prefers python3 over python
assert_eq!(c[1].binaries, &["python3", "python"]);
// node prefers node over nodejs
assert_eq!(c[2].binaries, &["node", "nodejs"]);
}
}

84
crates/worker/src/service.rs
View File

@@ -45,12 +45,32 @@ use crate::runtime::local::LocalRuntime;
use crate::runtime::native::NativeRuntime;
use crate::runtime::process::ProcessRuntime;
use crate::runtime::RuntimeRegistry;
use crate::runtime_detect::DetectedRuntime;
use crate::secrets::SecretManager;
use crate::version_verify;
use attune_common::repositories::runtime::RuntimeRepository;
use attune_common::repositories::List;
/// Controls how the worker initializes its runtime environment.
///
/// The standard `attune-worker` binary uses `Worker` mode with proactive
/// setup at startup, while the `attune-agent` binary uses `Agent` mode
/// with lazy (on-demand) initialization.
#[derive(Debug, Clone)]
pub enum StartupMode {
/// Full worker mode: proactive environment setup, full version
/// verification sweep at startup. Used by `attune-worker`.
Worker,
/// Agent mode: lazy environment setup (on first use), on-demand
/// version verification, auto-detected runtimes. Used by `attune-agent`.
Agent {
/// Runtimes detected by the auto-detection module.
detected_runtimes: Vec<DetectedRuntime>,
},
}
/// Message payload for execution.scheduled events
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExecutionScheduledPayload {
@@ -93,6 +113,10 @@ pub struct WorkerService {
/// Tracks cancellation requests that arrived before the in-memory token
/// for an execution had been registered.
pending_cancellations: Arc<Mutex<HashSet<i64>>>,
/// Controls whether this worker runs in full `Worker` mode (proactive
/// environment setup, full version verification) or `Agent` mode (lazy
/// setup, auto-detected runtimes).
startup_mode: StartupMode,
}
impl WorkerService {
@@ -402,9 +426,26 @@ impl WorkerService {
in_flight_tasks: Arc::new(Mutex::new(JoinSet::new())),
cancel_tokens: Arc::new(Mutex::new(HashMap::new())),
pending_cancellations: Arc::new(Mutex::new(HashSet::new())),
startup_mode: StartupMode::Worker,
})
}
/// Set agent-detected runtimes for inclusion in worker registration.
///
/// When the worker is started as `attune-agent`, the agent entrypoint
/// auto-detects available interpreters and passes them here. During
/// [`start()`](Self::start), the detection results are stored in the
/// worker's capabilities as `detected_interpreters` (structured JSON
/// with binary paths and versions) and the `agent_mode` flag is set.
///
/// This method is a no-op for the standard `attune-worker` binary.
pub fn with_detected_runtimes(mut self, runtimes: Vec<DetectedRuntime>) -> Self {
self.startup_mode = StartupMode::Agent {
detected_runtimes: runtimes,
};
self
}
/// Start the worker service
pub async fn start(&mut self) -> Result<()> {
info!("Starting Worker Service");
@@ -413,6 +454,21 @@ impl WorkerService {
let worker_id = {
let mut reg = self.registration.write().await;
reg.detect_capabilities(&self.config).await?;
// If running as an agent, store the detected interpreter metadata
// and set the agent_mode flag before registering.
if let StartupMode::Agent {
ref detected_runtimes,
} = self.startup_mode
{
reg.set_detected_runtimes(detected_runtimes.clone());
reg.set_agent_mode(true);
info!(
"Agent mode: {} detected interpreter(s) will be stored in capabilities",
detected_runtimes.len()
);
}
reg.register().await?
};
self.worker_id = Some(worker_id);
@@ -430,16 +486,26 @@ impl WorkerService {
})?;
info!("Worker-specific message queue infrastructure setup completed");
// Verify which runtime versions are available on this system.
// This updates the `available` flag in the database so that
// `select_best_version()` only considers genuinely present versions.
self.verify_runtime_versions().await;
match &self.startup_mode {
StartupMode::Worker => {
// Verify which runtime versions are available on this system.
// This updates the `available` flag in the database so that
// `select_best_version()` only considers genuinely present versions.
self.verify_runtime_versions().await;
// Proactively set up runtime environments for all registered packs.
// This runs before we start consuming execution messages so that
// environments are ready by the time the first execution arrives.
// Now version-aware: creates per-version environments where needed.
self.scan_and_setup_environments().await;
// Proactively set up runtime environments for all registered packs.
// This runs before we start consuming execution messages so that
// environments are ready by the time the first execution arrives.
// Now version-aware: creates per-version environments where needed.
self.scan_and_setup_environments().await;
}
StartupMode::Agent { .. } => {
// Skip proactive setup — will happen lazily on first execution
info!(
"Agent mode: deferring environment setup and version verification to first use"
);
}
}
// Start heartbeat
self.heartbeat.start().await?;