attune-system/attune
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

artifacts!

Browse Source
This commit is contained in:
David Culbreth
2026-03-03 13:42:41 -06:00
parent 5da940639a
commit 8299e5efcb
50 changed files with 4779 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

580
crates/api/src/routes/artifacts.rs
View File

@@ -2,6 +2,7 @@
//!
//! Provides endpoints for:
//! - CRUD operations on artifacts (metadata + data)
//! - File-backed version creation (execution writes file to shared volume)
//! - File upload (binary) and download for file-type artifacts
//! - JSON content versioning for structured artifacts
//! - Progress append for progress-type artifacts (streaming updates)
@@ -17,8 +18,9 @@ use axum::{
Json, Router,
};
use std::sync::Arc;
use tracing::warn;
use attune_common::models::enums::ArtifactType;
use attune_common::models::enums::{ArtifactType, ArtifactVisibility};
use attune_common::repositories::{
artifact::{
ArtifactRepository, ArtifactSearchFilters, ArtifactVersionRepository, CreateArtifactInput,
@@ -33,7 +35,8 @@ use crate::{
artifact::{
AppendProgressRequest, ArtifactQueryParams, ArtifactResponse, ArtifactSummary,
ArtifactVersionResponse, ArtifactVersionSummary, CreateArtifactRequest,
CreateVersionJsonRequest, SetDataRequest, UpdateArtifactRequest,
CreateFileVersionRequest, CreateVersionJsonRequest, SetDataRequest,
UpdateArtifactRequest,
},
common::{PaginatedResponse, PaginationParams},
ApiResponse, SuccessResponse,
@@ -66,6 +69,7 @@ pub async fn list_artifacts(
scope: query.scope,
owner: query.owner.clone(),
r#type: query.r#type,
visibility: query.visibility,
execution: query.execution,
name_contains: query.name.clone(),
limit: query.limit(),
@@ -175,11 +179,22 @@ pub async fn create_artifact(
)));
}
// Type-aware visibility default: progress artifacts are public by default
// (they're informational status indicators), everything else is private.
let visibility = request.visibility.unwrap_or_else(|| {
if request.r#type == ArtifactType::Progress {
ArtifactVisibility::Public
} else {
ArtifactVisibility::Private
}
});
let input = CreateArtifactInput {
r#ref: request.r#ref,
scope: request.scope,
owner: request.owner,
r#type: request.r#type,
visibility,
retention_policy: request.retention_policy,
retention_limit: request.retention_limit,
name: request.name,
@@ -229,6 +244,7 @@ pub async fn update_artifact(
scope: request.scope,
owner: request.owner,
r#type: request.r#type,
visibility: request.visibility,
retention_policy: request.retention_policy,
retention_limit: request.retention_limit,
name: request.name,
@@ -249,7 +265,7 @@ pub async fn update_artifact(
))
}
/// Delete an artifact (cascades to all versions)
/// Delete an artifact (cascades to all versions, including disk files)
#[utoipa::path(
delete,
path = "/api/v1/artifacts/{id}",
@@ -266,6 +282,22 @@ pub async fn delete_artifact(
State(state): State<Arc<AppState>>,
Path(id): Path<i64>,
) -> ApiResult<impl IntoResponse> {
let artifact = ArtifactRepository::find_by_id(&state.db, id)
.await?
.ok_or_else(|| ApiError::NotFound(format!("Artifact with ID {} not found", id)))?;
// Before deleting DB rows, clean up any file-backed versions on disk
let file_versions =
ArtifactVersionRepository::find_file_versions_by_artifact(&state.db, id).await?;
if !file_versions.is_empty() {
let artifacts_dir = &state.config.artifacts_dir;
cleanup_version_files(artifacts_dir, &file_versions);
// Also try to remove the artifact's parent directory if it's now empty
let ref_dir = ref_to_dir_path(&artifact.r#ref);
let full_ref_dir = std::path::Path::new(artifacts_dir).join(&ref_dir);
cleanup_empty_parents(&full_ref_dir, artifacts_dir);
}
let deleted = ArtifactRepository::delete(&state.db, id).await?;
if !deleted {
return Err(ApiError::NotFound(format!(
@@ -527,6 +559,7 @@ pub async fn create_version_json(
),
content: None,
content_json: Some(request.content),
file_path: None,
meta: request.meta,
created_by: request.created_by,
};
@@ -542,6 +575,108 @@ pub async fn create_version_json(
))
}
/// Create a new file-backed version (no file content in request).
///
/// This endpoint allocates a version number and computes a `file_path` on the
/// shared artifact volume. The caller (execution process) is expected to write
/// the file content directly to `$ATTUNE_ARTIFACTS_DIR/{file_path}` after
/// receiving the response. The worker finalizes `size_bytes` after execution.
///
/// Only applicable to file-type artifacts (FileBinary, FileDatatable, FileText, Log).
#[utoipa::path(
post,
path = "/api/v1/artifacts/{id}/versions/file",
tag = "artifacts",
params(("id" = i64, Path, description = "Artifact ID")),
request_body = CreateFileVersionRequest,
responses(
(status = 201, description = "File version allocated", body = inline(ApiResponse<ArtifactVersionResponse>)),
(status = 400, description = "Artifact type is not file-based"),
(status = 404, description = "Artifact not found"),
),
security(("bearer_auth" = []))
)]
pub async fn create_version_file(
RequireAuth(_user): RequireAuth,
State(state): State<Arc<AppState>>,
Path(id): Path<i64>,
Json(request): Json<CreateFileVersionRequest>,
) -> ApiResult<impl IntoResponse> {
let artifact = ArtifactRepository::find_by_id(&state.db, id)
.await?
.ok_or_else(|| ApiError::NotFound(format!("Artifact with ID {} not found", id)))?;
// Validate this is a file-type artifact
if !is_file_backed_type(artifact.r#type) {
return Err(ApiError::BadRequest(format!(
"Artifact '{}' is type {:?}, which does not support file-backed versions. \
Use POST /versions for JSON or POST /versions/upload for DB-stored files.",
artifact.r#ref, artifact.r#type,
)));
}
let content_type = request
.content_type
.unwrap_or_else(|| default_content_type_for_artifact(artifact.r#type));
// We need the version number to compute the file path. The DB function
// `next_artifact_version()` is called inside the INSERT, so we create the
// row first with file_path = NULL, then compute the path from the returned
// version number and update the row. This avoids a race condition where two
// concurrent requests could compute the same version number.
let input = CreateArtifactVersionInput {
artifact: id,
content_type: Some(content_type.clone()),
content: None,
content_json: None,
file_path: None, // Will be set in the update below
meta: request.meta,
created_by: request.created_by,
};
let version = ArtifactVersionRepository::create(&state.db, input).await?;
// Compute the file path from the artifact ref and version number
let file_path = compute_file_path(&artifact.r#ref, version.version, &content_type);
// Create the parent directory on disk
let artifacts_dir = &state.config.artifacts_dir;
let full_path = std::path::Path::new(artifacts_dir).join(&file_path);
if let Some(parent) = full_path.parent() {
tokio::fs::create_dir_all(parent).await.map_err(|e| {
ApiError::InternalServerError(format!(
"Failed to create artifact directory '{}': {}",
parent.display(),
e,
))
})?;
}
// Update the version row with the computed file_path
sqlx::query("UPDATE artifact_version SET file_path = $1 WHERE id = $2")
.bind(&file_path)
.execute(&state.db)
.await
.map_err(|e| {
ApiError::InternalServerError(format!(
"Failed to set file_path on version {}: {}",
version.id, e,
))
})?;
// Return the version with file_path populated
let mut response = ArtifactVersionResponse::from(version);
response.file_path = Some(file_path);
Ok((
StatusCode::CREATED,
Json(ApiResponse::with_message(
response,
"File version allocated — write content to $ATTUNE_ARTIFACTS_DIR/<file_path>",
)),
))
}
/// Upload a binary file as a new version (multipart/form-data)
///
/// The file is sent as a multipart form field named `file`. Optional fields:
@@ -656,6 +791,7 @@ pub async fn upload_version(
content_type: Some(resolved_ct),
content: Some(file_bytes),
content_json: None,
file_path: None,
meta,
created_by,
};
@@ -671,7 +807,10 @@ pub async fn upload_version(
))
}
/// Download the binary content of a specific version
/// Download the binary content of a specific version.
///
/// For file-backed versions, reads from the shared artifact volume on disk.
/// For DB-stored versions, reads from the BYTEA/JSON content column.
#[utoipa::path(
get,
path = "/api/v1/artifacts/{id}/versions/{version}/download",
@@ -695,69 +834,33 @@ pub async fn download_version(
.await?
.ok_or_else(|| ApiError::NotFound(format!("Artifact with ID {} not found", id)))?;
// First try without content (cheaper query) to check for file_path
let ver = ArtifactVersionRepository::find_by_version(&state.db, id, version)
.await?
.ok_or_else(|| {
ApiError::NotFound(format!("Version {} not found for artifact {}", version, id))
})?;
// File-backed version: read from disk
if let Some(ref file_path) = ver.file_path {
return serve_file_from_disk(
&state.config.artifacts_dir,
file_path,
&artifact.r#ref,
version,
ver.content_type.as_deref(),
)
.await;
}
// DB-stored version: need to fetch with content
let ver = ArtifactVersionRepository::find_by_version_with_content(&state.db, id, version)
.await?
.ok_or_else(|| {
ApiError::NotFound(format!("Version {} not found for artifact {}", version, id))
})?;
// For binary content
if let Some(bytes) = ver.content {
let ct = ver
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let filename = format!(
"{}_v{}.{}",
artifact.r#ref.replace('.', "_"),
version,
extension_from_content_type(&ct)
);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
)
.into_response());
}
// For JSON content, serialize and return
if let Some(json) = ver.content_json {
let bytes = serde_json::to_vec_pretty(&json).map_err(|e| {
ApiError::InternalServerError(format!("Failed to serialize JSON: {}", e))
})?;
let ct = ver
.content_type
.unwrap_or_else(|| "application/json".to_string());
let filename = format!("{}_v{}.json", artifact.r#ref.replace('.', "_"), version,);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
)
.into_response());
}
Err(ApiError::NotFound(format!(
"Version {} of artifact {} has no downloadable content",
version, id
)))
serve_db_content(&artifact.r#ref, version, &ver)
}
/// Download the latest version's content
@@ -781,72 +884,34 @@ pub async fn download_latest(
.await?
.ok_or_else(|| ApiError::NotFound(format!("Artifact with ID {} not found", id)))?;
let ver = ArtifactVersionRepository::find_latest_with_content(&state.db, id)
// First try without content (cheaper query) to check for file_path
let ver = ArtifactVersionRepository::find_latest(&state.db, id)
.await?
.ok_or_else(|| ApiError::NotFound(format!("No versions found for artifact {}", id)))?;
let version = ver.version;
// For binary content
if let Some(bytes) = ver.content {
let ct = ver
.content_type
.unwrap_or_else(|| "application/octet-stream".to_string());
let filename = format!(
"{}_v{}.{}",
artifact.r#ref.replace('.', "_"),
// File-backed version: read from disk
if let Some(ref file_path) = ver.file_path {
return serve_file_from_disk(
&state.config.artifacts_dir,
file_path,
&artifact.r#ref,
version,
extension_from_content_type(&ct)
);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
ver.content_type.as_deref(),
)
.into_response());
.await;
}
// For JSON content
if let Some(json) = ver.content_json {
let bytes = serde_json::to_vec_pretty(&json).map_err(|e| {
ApiError::InternalServerError(format!("Failed to serialize JSON: {}", e))
})?;
// DB-stored version: need to fetch with content
let ver = ArtifactVersionRepository::find_latest_with_content(&state.db, id)
.await?
.ok_or_else(|| ApiError::NotFound(format!("No versions found for artifact {}", id)))?;
let ct = ver
.content_type
.unwrap_or_else(|| "application/json".to_string());
let filename = format!("{}_v{}.json", artifact.r#ref.replace('.', "_"), version,);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
)
.into_response());
}
Err(ApiError::NotFound(format!(
"Latest version of artifact {} has no downloadable content",
id
)))
serve_db_content(&artifact.r#ref, ver.version, &ver)
}
/// Delete a specific version by version number
/// Delete a specific version by version number (including disk file if file-backed)
#[utoipa::path(
delete,
path = "/api/v1/artifacts/{id}/versions/{version}",
@@ -867,7 +932,7 @@ pub async fn delete_version(
Path((id, version)): Path<(i64, i32)>,
) -> ApiResult<impl IntoResponse> {
// Verify artifact exists
ArtifactRepository::find_by_id(&state.db, id)
let artifact = ArtifactRepository::find_by_id(&state.db, id)
.await?
.ok_or_else(|| ApiError::NotFound(format!("Artifact with ID {} not found", id)))?;
@@ -878,6 +943,25 @@ pub async fn delete_version(
ApiError::NotFound(format!("Version {} not found for artifact {}", version, id))
})?;
// Clean up disk file if file-backed
if let Some(ref file_path) = ver.file_path {
let artifacts_dir = &state.config.artifacts_dir;
let full_path = std::path::Path::new(artifacts_dir).join(file_path);
if full_path.exists() {
if let Err(e) = tokio::fs::remove_file(&full_path).await {
warn!(
"Failed to delete artifact file '{}': {}. DB row will still be deleted.",
full_path.display(),
e
);
}
}
// Try to clean up empty parent directories
let ref_dir = ref_to_dir_path(&artifact.r#ref);
let full_ref_dir = std::path::Path::new(artifacts_dir).join(&ref_dir);
cleanup_empty_parents(&full_ref_dir, artifacts_dir);
}
ArtifactVersionRepository::delete(&state.db, ver.id).await?;
Ok((
@@ -890,6 +974,212 @@ pub async fn delete_version(
// Helpers
// ============================================================================
/// Returns true for artifact types that should use file-backed storage on disk.
fn is_file_backed_type(artifact_type: ArtifactType) -> bool {
matches!(
artifact_type,
ArtifactType::FileBinary
| ArtifactType::FileText
| ArtifactType::FileDataTable
| ArtifactType::FileImage
)
}
/// Convert an artifact ref to a directory path by replacing dots with path separators.
/// e.g., "mypack.build_log" -> "mypack/build_log"
fn ref_to_dir_path(artifact_ref: &str) -> String {
artifact_ref.replace('.', "/")
}
/// Compute the relative file path for a file-backed artifact version.
///
/// Pattern: `{ref_slug}/v{version}.{ext}`
/// e.g., `mypack/build_log/v1.txt`
pub fn compute_file_path(artifact_ref: &str, version: i32, content_type: &str) -> String {
let ref_path = ref_to_dir_path(artifact_ref);
let ext = extension_from_content_type(content_type);
format!("{}/v{}.{}", ref_path, version, ext)
}
/// Return a sensible default content type for a given artifact type.
fn default_content_type_for_artifact(artifact_type: ArtifactType) -> String {
match artifact_type {
ArtifactType::FileText => "text/plain".to_string(),
ArtifactType::FileDataTable => "text/csv".to_string(),
ArtifactType::FileImage => "image/png".to_string(),
ArtifactType::FileBinary => "application/octet-stream".to_string(),
_ => "application/octet-stream".to_string(),
}
}
/// Serve a file-backed artifact version from disk.
async fn serve_file_from_disk(
artifacts_dir: &str,
file_path: &str,
artifact_ref: &str,
version: i32,
content_type: Option<&str>,
) -> ApiResult<axum::response::Response> {
let full_path = std::path::Path::new(artifacts_dir).join(file_path);
if !full_path.exists() {
return Err(ApiError::NotFound(format!(
"File for version {} of artifact '{}' not found on disk (expected at '{}')",
version, artifact_ref, file_path,
)));
}
let bytes = tokio::fs::read(&full_path).await.map_err(|e| {
ApiError::InternalServerError(format!(
"Failed to read artifact file '{}': {}",
full_path.display(),
e,
))
})?;
let ct = content_type
.unwrap_or("application/octet-stream")
.to_string();
let filename = format!(
"{}_v{}.{}",
artifact_ref.replace('.', "_"),
version,
extension_from_content_type(&ct),
);
Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
)
.into_response())
}
/// Serve a DB-stored artifact version (BYTEA or JSON content).
fn serve_db_content(
artifact_ref: &str,
version: i32,
ver: &attune_common::models::artifact_version::ArtifactVersion,
) -> ApiResult<axum::response::Response> {
// For binary content
if let Some(ref bytes) = ver.content {
let ct = ver
.content_type
.clone()
.unwrap_or_else(|| "application/octet-stream".to_string());
let filename = format!(
"{}_v{}.{}",
artifact_ref.replace('.', "_"),
version,
extension_from_content_type(&ct),
);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes.clone()),
)
.into_response());
}
// For JSON content, serialize and return
if let Some(ref json) = ver.content_json {
let bytes = serde_json::to_vec_pretty(json).map_err(|e| {
ApiError::InternalServerError(format!("Failed to serialize JSON: {}", e))
})?;
let ct = ver
.content_type
.clone()
.unwrap_or_else(|| "application/json".to_string());
let filename = format!("{}_v{}.json", artifact_ref.replace('.', "_"), version);
return Ok((
StatusCode::OK,
[
(header::CONTENT_TYPE, ct),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{}\"", filename),
),
],
Body::from(bytes),
)
.into_response());
}
Err(ApiError::NotFound(format!(
"Version {} of artifact '{}' has no downloadable content",
version, artifact_ref,
)))
}
/// Delete disk files for a set of file-backed artifact versions.
/// Logs warnings on failure but does not propagate errors.
fn cleanup_version_files(
artifacts_dir: &str,
versions: &[attune_common::models::artifact_version::ArtifactVersion],
) {
for ver in versions {
if let Some(ref file_path) = ver.file_path {
let full_path = std::path::Path::new(artifacts_dir).join(file_path);
if full_path.exists() {
if let Err(e) = std::fs::remove_file(&full_path) {
warn!(
"Failed to delete artifact file '{}': {}",
full_path.display(),
e,
);
}
}
}
}
}
/// Attempt to remove empty parent directories up to (but not including) the
/// artifacts_dir root. This is best-effort cleanup.
fn cleanup_empty_parents(dir: &std::path::Path, stop_at: &str) {
let stop_path = std::path::Path::new(stop_at);
let mut current = dir.to_path_buf();
while current != stop_path && current.starts_with(stop_path) {
match std::fs::read_dir(&current) {
Ok(mut entries) => {
if entries.next().is_some() {
// Directory is not empty, stop climbing
break;
}
if let Err(e) = std::fs::remove_dir(&current) {
warn!(
"Failed to remove empty directory '{}': {}",
current.display(),
e,
);
break;
}
}
Err(_) => break,
}
match current.parent() {
Some(parent) => current = parent.to_path_buf(),
None => break,
}
}
}
/// Derive a simple file extension from a MIME content type
fn extension_from_content_type(ct: &str) -> &str {
match ct {
@@ -944,6 +1234,7 @@ pub fn routes() -> Router<Arc<AppState>> {
)
.route("/artifacts/{id}/versions/latest", get(get_latest_version))
.route("/artifacts/{id}/versions/upload", post(upload_version))
.route("/artifacts/{id}/versions/file", post(create_version_file))
.route(
"/artifacts/{id}/versions/{version}",
get(get_version).delete(delete_version),
@@ -975,4 +1266,61 @@ mod tests {
assert_eq!(extension_from_content_type("image/png"), "png");
assert_eq!(extension_from_content_type("unknown/type"), "bin");
}
#[test]
fn test_compute_file_path() {
assert_eq!(
compute_file_path("mypack.build_log", 1, "text/plain"),
"mypack/build_log/v1.txt"
);
assert_eq!(
compute_file_path("mypack.build_log", 3, "application/json"),
"mypack/build_log/v3.json"
);
assert_eq!(
compute_file_path("core.test.results", 2, "text/csv"),
"core/test/results/v2.csv"
);
assert_eq!(
compute_file_path("simple", 1, "application/octet-stream"),
"simple/v1.bin"
);
}
#[test]
fn test_ref_to_dir_path() {
assert_eq!(ref_to_dir_path("mypack.build_log"), "mypack/build_log");
assert_eq!(ref_to_dir_path("simple"), "simple");
assert_eq!(ref_to_dir_path("a.b.c.d"), "a/b/c/d");
}
#[test]
fn test_is_file_backed_type() {
assert!(is_file_backed_type(ArtifactType::FileBinary));
assert!(is_file_backed_type(ArtifactType::FileText));
assert!(is_file_backed_type(ArtifactType::FileDataTable));
assert!(is_file_backed_type(ArtifactType::FileImage));
assert!(!is_file_backed_type(ArtifactType::Progress));
assert!(!is_file_backed_type(ArtifactType::Url));
}
#[test]
fn test_default_content_type_for_artifact() {
assert_eq!(
default_content_type_for_artifact(ArtifactType::FileText),
"text/plain"
);
assert_eq!(
default_content_type_for_artifact(ArtifactType::FileDataTable),
"text/csv"
);
assert_eq!(
default_content_type_for_artifact(ArtifactType::FileImage),
"image/png"
);
assert_eq!(
default_content_type_for_artifact(ArtifactType::FileBinary),
"application/octet-stream"
);
}
}