distributable, please

docker/distributable/migrations/20250101000001_initial_setup.sql

@@ -0,0 +1,230 @@
+-- Migration: Initial Setup
+-- Description: Creates the attune schema, enums, and shared database functions
+-- Version: 20250101000001
+
+-- ============================================================================
+-- EXTENSIONS
+-- ============================================================================
+
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+-- ============================================================================
+-- ENUM TYPES
+-- ============================================================================
+
+-- WorkerType enum
+DO $$ BEGIN
+    CREATE TYPE worker_type_enum AS ENUM (
+        'local',
+        'remote',
+        'container'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE worker_type_enum IS 'Type of worker deployment';
+
+-- WorkerRole enum
+DO $$ BEGIN
+    CREATE TYPE worker_role_enum AS ENUM (
+        'action',
+        'sensor',
+        'hybrid'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE worker_role_enum IS 'Role of worker (action executor, sensor, or both)';
+
+
+-- WorkerStatus enum
+DO $$ BEGIN
+    CREATE TYPE worker_status_enum AS ENUM (
+        'active',
+        'inactive',
+        'busy',
+        'error'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE worker_status_enum IS 'Worker operational status';
+
+-- EnforcementStatus enum
+DO $$ BEGIN
+    CREATE TYPE enforcement_status_enum AS ENUM (
+        'created',
+        'processed',
+        'disabled'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE enforcement_status_enum IS 'Enforcement processing status';
+
+-- EnforcementCondition enum
+DO $$ BEGIN
+    CREATE TYPE enforcement_condition_enum AS ENUM (
+        'any',
+        'all'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE enforcement_condition_enum IS 'Logical operator for conditions (OR/AND)';
+
+-- ExecutionStatus enum
+DO $$ BEGIN
+    CREATE TYPE execution_status_enum AS ENUM (
+        'requested',
+        'scheduling',
+        'scheduled',
+        'running',
+        'completed',
+        'failed',
+        'canceling',
+        'cancelled',
+        'timeout',
+        'abandoned'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE execution_status_enum IS 'Execution lifecycle status';
+
+-- InquiryStatus enum
+DO $$ BEGIN
+    CREATE TYPE inquiry_status_enum AS ENUM (
+        'pending',
+        'responded',
+        'timeout',
+        'cancelled'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE inquiry_status_enum IS 'Inquiry lifecycle status';
+
+-- PolicyMethod enum
+DO $$ BEGIN
+    CREATE TYPE policy_method_enum AS ENUM (
+        'cancel',
+        'enqueue'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE policy_method_enum IS 'Policy enforcement method';
+
+-- OwnerType enum
+DO $$ BEGIN
+    CREATE TYPE owner_type_enum AS ENUM (
+        'system',
+        'identity',
+        'pack',
+        'action',
+        'sensor'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE owner_type_enum IS 'Type of resource owner';
+
+-- NotificationState enum
+DO $$ BEGIN
+    CREATE TYPE notification_status_enum AS ENUM (
+        'created',
+        'queued',
+        'processing',
+        'error'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE notification_status_enum IS 'Notification processing state';
+
+-- ArtifactType enum
+DO $$ BEGIN
+    CREATE TYPE artifact_type_enum AS ENUM (
+        'file_binary',
+        'file_datatable',
+        'file_image',
+        'file_text',
+        'other',
+        'progress',
+        'url'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE artifact_type_enum IS 'Type of artifact';
+
+-- RetentionPolicyType enum
+DO $$ BEGIN
+    CREATE TYPE artifact_retention_enum AS ENUM (
+        'versions',
+        'days',
+        'hours',
+        'minutes'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE artifact_retention_enum IS 'Type of retention policy';
+
+-- ArtifactVisibility enum
+DO $$ BEGIN
+    CREATE TYPE artifact_visibility_enum AS ENUM (
+        'public',
+        'private'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE artifact_visibility_enum IS 'Visibility of an artifact (public = viewable by all users, private = scoped by owner)';
+
+
+-- PackEnvironmentStatus enum
+DO $$ BEGIN
+    CREATE TYPE pack_environment_status_enum AS ENUM (
+        'pending',
+        'installing',
+        'ready',
+        'failed',
+        'outdated'
+    );
+EXCEPTION
+    WHEN duplicate_object THEN null;
+END $$;
+
+COMMENT ON TYPE pack_environment_status_enum IS 'Status of pack runtime environment installation';
+
+-- ============================================================================
+-- SHARED FUNCTIONS
+-- ============================================================================
+
+-- Function to automatically update the 'updated' timestamp
+CREATE OR REPLACE FUNCTION update_updated_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated = NOW();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION update_updated_column() IS 'Automatically updates the updated timestamp on row modification';

docker/distributable/migrations/20250101000002_pack_system.sql

@@ -0,0 +1,262 @@
+-- Migration: Pack System
+-- Description: Creates pack, runtime, and runtime_version tables
+-- Version: 20250101000002
+
+-- ============================================================================
+-- PACK TABLE
+-- ============================================================================
+
+CREATE TABLE pack (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    label TEXT NOT NULL,
+    description TEXT,
+    version TEXT NOT NULL,
+    conf_schema JSONB NOT NULL DEFAULT '{}'::jsonb,
+    config JSONB NOT NULL DEFAULT '{}'::jsonb,
+    meta JSONB NOT NULL DEFAULT '{}'::jsonb,
+    tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+    runtime_deps TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+    dependencies TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+    is_standard BOOLEAN NOT NULL DEFAULT FALSE,
+    installers JSONB DEFAULT '[]'::jsonb,
+
+    -- Installation metadata (nullable for non-installed packs)
+    source_type TEXT,
+    source_url TEXT,
+    source_ref TEXT,
+    checksum TEXT,
+    checksum_verified BOOLEAN DEFAULT FALSE,
+    installed_at TIMESTAMPTZ,
+    installed_by BIGINT,
+    installation_method TEXT,
+    storage_path TEXT,
+
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT pack_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT pack_ref_format CHECK (ref ~ '^[a-z][a-z0-9_-]+$'),
+    CONSTRAINT pack_version_semver CHECK (
+        version ~ '^\d+\.\d+\.\d+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'
+    )
+);
+
+-- Indexes
+CREATE INDEX idx_pack_ref ON pack(ref);
+CREATE INDEX idx_pack_created ON pack(created DESC);
+CREATE INDEX idx_pack_is_standard ON pack(is_standard) WHERE is_standard = TRUE;
+CREATE INDEX idx_pack_is_standard_created ON pack(is_standard, created DESC);
+CREATE INDEX idx_pack_version_created ON pack(version, created DESC);
+CREATE INDEX idx_pack_config_gin ON pack USING GIN (config);
+CREATE INDEX idx_pack_meta_gin ON pack USING GIN (meta);
+CREATE INDEX idx_pack_tags_gin ON pack USING GIN (tags);
+CREATE INDEX idx_pack_runtime_deps_gin ON pack USING GIN (runtime_deps);
+CREATE INDEX idx_pack_dependencies_gin ON pack USING GIN (dependencies);
+CREATE INDEX idx_pack_installed_at ON pack(installed_at DESC) WHERE installed_at IS NOT NULL;
+CREATE INDEX idx_pack_installed_by ON pack(installed_by) WHERE installed_by IS NOT NULL;
+CREATE INDEX idx_pack_source_type ON pack(source_type) WHERE source_type IS NOT NULL;
+
+-- Trigger
+CREATE TRIGGER update_pack_updated
+    BEFORE UPDATE ON pack
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE pack IS 'Packs bundle related automation components';
+COMMENT ON COLUMN pack.ref IS 'Unique pack reference identifier (e.g., "slack", "github")';
+COMMENT ON COLUMN pack.label IS 'Human-readable pack name';
+COMMENT ON COLUMN pack.version IS 'Semantic version of the pack';
+COMMENT ON COLUMN pack.conf_schema IS 'JSON schema for pack configuration';
+COMMENT ON COLUMN pack.config IS 'Pack configuration values';
+COMMENT ON COLUMN pack.meta IS 'Pack metadata';
+COMMENT ON COLUMN pack.runtime_deps IS 'Array of required runtime references (e.g., shell, python, nodejs)';
+COMMENT ON COLUMN pack.dependencies IS 'Array of required pack references (e.g., core, utils)';
+COMMENT ON COLUMN pack.is_standard IS 'Whether this is a core/built-in pack';
+COMMENT ON COLUMN pack.source_type IS 'Installation source type (e.g., "git", "local", "registry")';
+COMMENT ON COLUMN pack.source_url IS 'URL or path where pack was installed from';
+COMMENT ON COLUMN pack.source_ref IS 'Git ref, version tag, or other source reference';
+COMMENT ON COLUMN pack.checksum IS 'Content checksum for verification';
+COMMENT ON COLUMN pack.checksum_verified IS 'Whether checksum has been verified';
+COMMENT ON COLUMN pack.installed_at IS 'Timestamp when pack was installed';
+COMMENT ON COLUMN pack.installed_by IS 'Identity ID of user who installed the pack';
+COMMENT ON COLUMN pack.installation_method IS 'Method used for installation (e.g., "cli", "api", "auto")';
+COMMENT ON COLUMN pack.storage_path IS 'Filesystem path where pack files are stored';
+
+-- ============================================================================
+-- RUNTIME TABLE
+-- ============================================================================
+
+CREATE TABLE runtime (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT,
+    description TEXT,
+    name TEXT NOT NULL,
+    aliases TEXT[] NOT NULL DEFAULT '{}'::text[],
+
+    distributions JSONB NOT NULL,
+    installation JSONB,
+    installers JSONB DEFAULT '[]'::jsonb,
+
+    -- Execution configuration: describes how to execute actions using this runtime,
+    -- how to create isolated environments, and how to install dependencies.
+    --
+    -- Structure:
+    -- {
+    --   "interpreter": {
+    --     "binary": "python3",          -- interpreter binary name or path
+    --     "args": [],                   -- additional args before the action file
+    --     "file_extension": ".py"       -- file extension this runtime handles
+    --   },
+    --   "environment": {               -- optional: isolated environment config
+    --     "env_type": "virtualenv",     -- "virtualenv", "node_modules", "none"
+    --     "dir_name": ".venv",          -- directory name relative to pack dir
+    --     "create_command": ["python3", "-m", "venv", "{env_dir}"],
+    --     "interpreter_path": "{env_dir}/bin/python3"  -- overrides interpreter.binary
+    --   },
+    --   "dependencies": {              -- optional: dependency management config
+    --     "manifest_file": "requirements.txt",
+    --     "install_command": ["{interpreter}", "-m", "pip", "install", "-r", "{manifest_path}"]
+    --   }
+    -- }
+    --
+    -- Template variables:
+    --   {pack_dir}      - absolute path to the pack directory
+    --   {env_dir}       - resolved environment directory (pack_dir/dir_name)
+    --   {interpreter}   - resolved interpreter path
+    --   {action_file}   - absolute path to the action script file
+    --   {manifest_path} - absolute path to the dependency manifest file
+    execution_config JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+    -- Whether this runtime was auto-registered by an agent
+    -- (vs. loaded from a pack's YAML file during pack registration)
+    auto_detected BOOLEAN NOT NULL DEFAULT FALSE,
+
+    -- Detection metadata for auto-discovered runtimes.
+    -- Stores how the agent discovered this runtime (binary path, version, etc.)
+    -- enables re-verification on restart.
+    -- Example: { "detected_path": "/usr/bin/ruby", "detected_name": "ruby",
+    --            "detected_version": "3.3.0" }
+    detection_config JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT runtime_ref_lowercase CHECK (ref = LOWER(ref))
+);
+
+-- Indexes
+CREATE INDEX idx_runtime_ref ON runtime(ref);
+CREATE INDEX idx_runtime_pack ON runtime(pack);
+CREATE INDEX idx_runtime_created ON runtime(created DESC);
+CREATE INDEX idx_runtime_name ON runtime(name);
+CREATE INDEX idx_runtime_verification ON runtime USING GIN ((distributions->'verification'));
+CREATE INDEX idx_runtime_execution_config ON runtime USING GIN (execution_config);
+CREATE INDEX idx_runtime_auto_detected ON runtime(auto_detected);
+CREATE INDEX idx_runtime_detection_config ON runtime USING GIN (detection_config);
+CREATE INDEX idx_runtime_aliases ON runtime USING GIN (aliases);
+
+-- Trigger
+CREATE TRIGGER update_runtime_updated
+    BEFORE UPDATE ON runtime
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE runtime IS 'Runtime environments for executing actions and sensors (unified)';
+COMMENT ON COLUMN runtime.ref IS 'Unique runtime reference (format: pack.name, e.g., core.python)';
+COMMENT ON COLUMN runtime.name IS 'Runtime name (e.g., "Python", "Node.js", "Shell")';
+COMMENT ON COLUMN runtime.aliases IS 'Lowercase alias names for this runtime (e.g., ["ruby", "rb"] for the Ruby runtime). Used for alias-aware matching during auto-detection and scheduling.';
+COMMENT ON COLUMN runtime.distributions IS 'Runtime distribution metadata including verification commands, version requirements, and capabilities';
+COMMENT ON COLUMN runtime.installation IS 'Installation requirements and instructions including package managers and setup steps';
+COMMENT ON COLUMN runtime.installers IS 'Array of installer actions to create pack-specific runtime environments. Each installer defines commands to set up isolated environments (e.g., Python venv, npm install).';
+COMMENT ON COLUMN runtime.execution_config IS 'Execution configuration: interpreter, environment setup, and dependency management. Drives how the worker executes actions and how pack install sets up environments.';
+COMMENT ON COLUMN runtime.auto_detected IS 'Whether this runtime was auto-registered by an agent (true) vs. loaded from a pack YAML (false)';
+COMMENT ON COLUMN runtime.detection_config IS 'Detection metadata for auto-discovered runtimes: binaries probed, version regex, detected path/version';
+
+-- ============================================================================
+-- RUNTIME VERSION TABLE
+-- ============================================================================
+
+CREATE TABLE runtime_version (
+    id BIGSERIAL PRIMARY KEY,
+    runtime BIGINT NOT NULL REFERENCES runtime(id) ON DELETE CASCADE,
+    runtime_ref TEXT NOT NULL,
+
+    -- Semantic version string (e.g., "3.12.1", "20.11.0")
+    version TEXT NOT NULL,
+
+    -- Individual version components for efficient range queries.
+    -- Nullable because some runtimes may use non-numeric versioning.
+    version_major INT,
+    version_minor INT,
+    version_patch INT,
+
+    -- Complete execution configuration for this specific version.
+    -- This is NOT a diff/override — it is a full standalone config that can
+    -- replace the parent runtime's execution_config when this version is selected.
+    -- Structure is identical to runtime.execution_config (RuntimeExecutionConfig).
+    execution_config JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+    -- Version-specific distribution/verification metadata.
+    -- Structure mirrors runtime.distributions but with version-specific commands.
+    -- Example: verification commands that check for a specific binary like python3.12.
+    distributions JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+    -- Whether this version is the default for the parent runtime.
+    -- At most one version per runtime should be marked as default.
+    is_default BOOLEAN NOT NULL DEFAULT FALSE,
+
+    -- Whether this version has been verified as available on the current system.
+    available BOOLEAN NOT NULL DEFAULT TRUE,
+
+    -- When this version was last verified (via running verification commands).
+    verified_at TIMESTAMPTZ,
+
+    -- Arbitrary version-specific metadata (e.g., EOL date, release notes URL,
+    -- feature flags, platform-specific notes).
+    meta JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT runtime_version_unique UNIQUE(runtime, version)
+);
+
+-- Indexes
+CREATE INDEX idx_runtime_version_runtime ON runtime_version(runtime);
+CREATE INDEX idx_runtime_version_runtime_ref ON runtime_version(runtime_ref);
+CREATE INDEX idx_runtime_version_version ON runtime_version(version);
+CREATE INDEX idx_runtime_version_available ON runtime_version(available) WHERE available = TRUE;
+CREATE INDEX idx_runtime_version_is_default ON runtime_version(is_default) WHERE is_default = TRUE;
+CREATE INDEX idx_runtime_version_components ON runtime_version(runtime, version_major, version_minor, version_patch);
+CREATE INDEX idx_runtime_version_created ON runtime_version(created DESC);
+CREATE INDEX idx_runtime_version_execution_config ON runtime_version USING GIN (execution_config);
+CREATE INDEX idx_runtime_version_meta ON runtime_version USING GIN (meta);
+
+-- Trigger
+CREATE TRIGGER update_runtime_version_updated
+    BEFORE UPDATE ON runtime_version
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE runtime_version IS 'Specific versions of a runtime (e.g., Python 3.11, 3.12) with version-specific execution configuration';
+COMMENT ON COLUMN runtime_version.runtime IS 'Parent runtime this version belongs to';
+COMMENT ON COLUMN runtime_version.runtime_ref IS 'Parent runtime ref (e.g., core.python) for display/filtering';
+COMMENT ON COLUMN runtime_version.version IS 'Semantic version string (e.g., "3.12.1", "20.11.0")';
+COMMENT ON COLUMN runtime_version.version_major IS 'Major version component for efficient range queries';
+COMMENT ON COLUMN runtime_version.version_minor IS 'Minor version component for efficient range queries';
+COMMENT ON COLUMN runtime_version.version_patch IS 'Patch version component for efficient range queries';
+COMMENT ON COLUMN runtime_version.execution_config IS 'Complete execution configuration for this version (same structure as runtime.execution_config)';
+COMMENT ON COLUMN runtime_version.distributions IS 'Version-specific distribution/verification metadata';
+COMMENT ON COLUMN runtime_version.is_default IS 'Whether this is the default version for the parent runtime (at most one per runtime)';
+COMMENT ON COLUMN runtime_version.available IS 'Whether this version has been verified as available on the system';
+COMMENT ON COLUMN runtime_version.verified_at IS 'Timestamp of last availability verification';
+COMMENT ON COLUMN runtime_version.meta IS 'Arbitrary version-specific metadata';

docker/distributable/migrations/20250101000003_identity_and_auth.sql

@@ -0,0 +1,223 @@
+-- Migration: Identity and Authentication
+-- Description: Creates identity, permission, and policy tables
+-- Version: 20250101000002
+
+-- ============================================================================
+-- IDENTITY TABLE
+-- ============================================================================
+
+CREATE TABLE identity (
+    id BIGSERIAL PRIMARY KEY,
+    login TEXT NOT NULL UNIQUE,
+    display_name TEXT,
+    password_hash TEXT,
+    attributes JSONB NOT NULL DEFAULT '{}'::jsonb,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_identity_login ON identity(login);
+CREATE INDEX idx_identity_created ON identity(created DESC);
+CREATE INDEX idx_identity_password_hash ON identity(password_hash) WHERE password_hash IS NOT NULL;
+CREATE INDEX idx_identity_attributes_gin ON identity USING GIN (attributes);
+
+-- Trigger
+CREATE TRIGGER update_identity_updated
+    BEFORE UPDATE ON identity
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE identity IS 'Identities represent users or service accounts';
+COMMENT ON COLUMN identity.login IS 'Unique login identifier';
+COMMENT ON COLUMN identity.display_name IS 'Human-readable name';
+COMMENT ON COLUMN identity.password_hash IS 'Argon2 hashed password for authentication (NULL for service accounts or external auth)';
+COMMENT ON COLUMN identity.attributes IS 'Custom attributes (email, groups, etc.)';
+
+-- ============================================================================
+-- ADD FOREIGN KEY CONSTRAINTS TO EXISTING TABLES
+-- ============================================================================
+
+-- Add foreign key constraint for pack.installed_by now that identity table exists
+ALTER TABLE pack
+    ADD CONSTRAINT fk_pack_installed_by
+    FOREIGN KEY (installed_by)
+    REFERENCES identity(id)
+    ON DELETE SET NULL;
+
+-- ============================================================================
+
+-- ============================================================================
+-- PERMISSION_SET TABLE
+-- ============================================================================
+
+CREATE TABLE permission_set (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT,
+    label TEXT,
+    description TEXT,
+    grants JSONB NOT NULL DEFAULT '[]'::jsonb,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT permission_set_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT permission_set_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$')
+);
+
+-- Indexes
+CREATE INDEX idx_permission_set_ref ON permission_set(ref);
+CREATE INDEX idx_permission_set_pack ON permission_set(pack);
+CREATE INDEX idx_permission_set_created ON permission_set(created DESC);
+
+-- Trigger
+CREATE TRIGGER update_permission_set_updated
+    BEFORE UPDATE ON permission_set
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE permission_set IS 'Permission sets group permissions together (like roles)';
+COMMENT ON COLUMN permission_set.ref IS 'Unique permission set reference (format: pack.name)';
+COMMENT ON COLUMN permission_set.label IS 'Human-readable name';
+COMMENT ON COLUMN permission_set.grants IS 'Array of permission grants';
+
+-- ============================================================================
+
+-- ============================================================================
+-- PERMISSION_ASSIGNMENT TABLE
+-- ============================================================================
+
+CREATE TABLE permission_assignment (
+    id BIGSERIAL PRIMARY KEY,
+    identity BIGINT NOT NULL REFERENCES identity(id) ON DELETE CASCADE,
+    permset BIGINT NOT NULL REFERENCES permission_set(id) ON DELETE CASCADE,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Unique constraint to prevent duplicate assignments
+    CONSTRAINT unique_identity_permset UNIQUE (identity, permset)
+);
+
+-- Indexes
+CREATE INDEX idx_permission_assignment_identity ON permission_assignment(identity);
+CREATE INDEX idx_permission_assignment_permset ON permission_assignment(permset);
+CREATE INDEX idx_permission_assignment_created ON permission_assignment(created DESC);
+CREATE INDEX idx_permission_assignment_identity_created ON permission_assignment(identity, created DESC);
+CREATE INDEX idx_permission_assignment_permset_created ON permission_assignment(permset, created DESC);
+
+-- Comments
+COMMENT ON TABLE permission_assignment IS 'Links identities to permission sets (many-to-many)';
+COMMENT ON COLUMN permission_assignment.identity IS 'Identity being granted permissions';
+COMMENT ON COLUMN permission_assignment.permset IS 'Permission set being assigned';
+
+-- ============================================================================
+
+ALTER TABLE identity
+    ADD COLUMN frozen BOOLEAN NOT NULL DEFAULT false;
+
+CREATE INDEX idx_identity_frozen ON identity(frozen);
+
+COMMENT ON COLUMN identity.frozen IS 'If true, authentication is blocked for this identity';
+
+CREATE TABLE identity_role_assignment (
+    id BIGSERIAL PRIMARY KEY,
+    identity BIGINT NOT NULL REFERENCES identity(id) ON DELETE CASCADE,
+    role TEXT NOT NULL,
+    source TEXT NOT NULL DEFAULT 'manual',
+    managed BOOLEAN NOT NULL DEFAULT false,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT unique_identity_role_assignment UNIQUE (identity, role)
+);
+
+CREATE INDEX idx_identity_role_assignment_identity
+    ON identity_role_assignment(identity);
+CREATE INDEX idx_identity_role_assignment_role
+    ON identity_role_assignment(role);
+CREATE INDEX idx_identity_role_assignment_source
+    ON identity_role_assignment(source);
+
+CREATE TRIGGER update_identity_role_assignment_updated
+    BEFORE UPDATE ON identity_role_assignment
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+COMMENT ON TABLE identity_role_assignment IS 'Links identities to role labels from manual assignment or external identity providers';
+COMMENT ON COLUMN identity_role_assignment.role IS 'Opaque role/group label (e.g. IDP group name)';
+COMMENT ON COLUMN identity_role_assignment.source IS 'Where the role assignment originated (manual, oidc, ldap, sync, etc.)';
+COMMENT ON COLUMN identity_role_assignment.managed IS 'True when the assignment is managed by external sync and should not be edited manually';
+
+CREATE TABLE permission_set_role_assignment (
+    id BIGSERIAL PRIMARY KEY,
+    permset BIGINT NOT NULL REFERENCES permission_set(id) ON DELETE CASCADE,
+    role TEXT NOT NULL,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT unique_permission_set_role_assignment UNIQUE (permset, role)
+);
+
+CREATE INDEX idx_permission_set_role_assignment_permset
+    ON permission_set_role_assignment(permset);
+CREATE INDEX idx_permission_set_role_assignment_role
+    ON permission_set_role_assignment(role);
+
+COMMENT ON TABLE permission_set_role_assignment IS 'Links permission sets to role labels for role-based grant expansion';
+COMMENT ON COLUMN permission_set_role_assignment.role IS 'Opaque role/group label associated with the permission set';
+
+-- ============================================================================
+
+-- ============================================================================
+-- POLICY TABLE
+-- ============================================================================
+
+CREATE TABLE policy (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT,
+    action BIGINT, -- Forward reference to action table, will add constraint in next migration
+    action_ref TEXT,
+    parameters TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+    method policy_method_enum NOT NULL,
+    threshold INTEGER NOT NULL,
+    name TEXT NOT NULL,
+    description TEXT,
+    tags TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT policy_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT policy_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$'),
+    CONSTRAINT policy_threshold_positive CHECK (threshold > 0)
+);
+
+-- Indexes
+CREATE INDEX idx_policy_ref ON policy(ref);
+CREATE INDEX idx_policy_pack ON policy(pack);
+CREATE INDEX idx_policy_action ON policy(action);
+CREATE INDEX idx_policy_created ON policy(created DESC);
+CREATE INDEX idx_policy_action_created ON policy(action, created DESC);
+CREATE INDEX idx_policy_pack_created ON policy(pack, created DESC);
+CREATE INDEX idx_policy_parameters_gin ON policy USING GIN (parameters);
+CREATE INDEX idx_policy_tags_gin ON policy USING GIN (tags);
+
+-- Trigger
+CREATE TRIGGER update_policy_updated
+    BEFORE UPDATE ON policy
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE policy IS 'Policies define execution controls (rate limiting, concurrency)';
+COMMENT ON COLUMN policy.ref IS 'Unique policy reference (format: pack.name)';
+COMMENT ON COLUMN policy.action IS 'Action this policy applies to';
+COMMENT ON COLUMN policy.parameters IS 'Parameter names used for policy grouping';
+COMMENT ON COLUMN policy.method IS 'How to handle policy violations (cancel/enqueue)';
+COMMENT ON COLUMN policy.threshold IS 'Numeric limit (e.g., max concurrent executions)';
+
+-- ============================================================================

docker/distributable/migrations/20250101000004_trigger_sensor_event_rule.sql

@@ -0,0 +1,290 @@
+-- Migration: Event System and Actions
+-- Description: Creates trigger, sensor, event, enforcement, and action tables
+--              with runtime version constraint support. Includes webhook key
+--              generation function used by webhook management functions in 000007.
+--
+--              NOTE: The event and enforcement tables are converted to TimescaleDB
+--              hypertables in migration 000009. Hypertables cannot be the target of
+--              FK constraints, so enforcement.event is a plain BIGINT with no FK.
+--              FKs *from* hypertables to regular tables (e.g., event.trigger → trigger,
+--              enforcement.rule → rule) are supported by TimescaleDB 2.x and are kept.
+-- Version: 20250101000004
+
+-- ============================================================================
+-- WEBHOOK KEY GENERATION
+-- ============================================================================
+
+-- Generates a unique webhook key in the format: wh_<32 random hex chars>
+-- Used by enable_trigger_webhook() and regenerate_trigger_webhook_key() in 000007.
+CREATE OR REPLACE FUNCTION generate_webhook_key()
+RETURNS VARCHAR(64) AS $$
+BEGIN
+    RETURN 'wh_' || encode(gen_random_bytes(16), 'hex');
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION generate_webhook_key() IS 'Generates a unique webhook key (format: wh_<32 hex chars>) for trigger webhook authentication';
+
+-- ============================================================================
+-- TRIGGER TABLE
+-- ============================================================================
+
+CREATE TABLE trigger (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT,
+    label TEXT NOT NULL,
+    description TEXT,
+    enabled BOOLEAN NOT NULL DEFAULT TRUE,
+    is_adhoc BOOLEAN DEFAULT false NOT NULL,
+    param_schema JSONB,
+    out_schema JSONB,
+    webhook_enabled BOOLEAN NOT NULL DEFAULT FALSE,
+    webhook_key VARCHAR(64) UNIQUE,
+    webhook_config JSONB DEFAULT '{}'::jsonb,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT trigger_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT trigger_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$')
+);
+
+-- Indexes
+CREATE INDEX idx_trigger_ref ON trigger(ref);
+CREATE INDEX idx_trigger_pack ON trigger(pack);
+CREATE INDEX idx_trigger_enabled ON trigger(enabled) WHERE enabled = TRUE;
+CREATE INDEX idx_trigger_created ON trigger(created DESC);
+CREATE INDEX idx_trigger_pack_enabled ON trigger(pack, enabled);
+CREATE INDEX idx_trigger_webhook_key ON trigger(webhook_key) WHERE webhook_key IS NOT NULL;
+CREATE INDEX idx_trigger_enabled_created ON trigger(enabled, created DESC) WHERE enabled = TRUE;
+
+-- Trigger
+CREATE TRIGGER update_trigger_updated
+    BEFORE UPDATE ON trigger
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE trigger IS 'Trigger definitions that can activate rules';
+COMMENT ON COLUMN trigger.ref IS 'Unique trigger reference (format: pack.name)';
+COMMENT ON COLUMN trigger.label IS 'Human-readable trigger name';
+COMMENT ON COLUMN trigger.enabled IS 'Whether this trigger is active';
+COMMENT ON COLUMN trigger.param_schema IS 'JSON schema defining the expected configuration parameters when this trigger is used';
+COMMENT ON COLUMN trigger.out_schema IS 'JSON schema defining the structure of event payloads generated by this trigger';
+
+-- ============================================================================
+
+
+-- ============================================================================
+-- SENSOR TABLE
+-- ============================================================================
+
+CREATE TABLE sensor (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT,
+    label TEXT NOT NULL,
+    description TEXT,
+    entrypoint TEXT NOT NULL,
+    runtime BIGINT NOT NULL REFERENCES runtime(id) ON DELETE CASCADE,
+    runtime_ref TEXT NOT NULL,
+    trigger BIGINT NOT NULL REFERENCES trigger(id) ON DELETE CASCADE,
+    trigger_ref TEXT NOT NULL,
+    enabled BOOLEAN NOT NULL,
+    is_adhoc BOOLEAN NOT NULL DEFAULT FALSE,
+    param_schema JSONB,
+    config JSONB,
+    runtime_version_constraint TEXT,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT sensor_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT sensor_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$')
+);
+
+-- Indexes
+CREATE INDEX idx_sensor_ref ON sensor(ref);
+CREATE INDEX idx_sensor_pack ON sensor(pack);
+CREATE INDEX idx_sensor_runtime ON sensor(runtime);
+CREATE INDEX idx_sensor_trigger ON sensor(trigger);
+CREATE INDEX idx_sensor_enabled ON sensor(enabled) WHERE enabled = TRUE;
+CREATE INDEX idx_sensor_is_adhoc ON sensor(is_adhoc) WHERE is_adhoc = true;
+CREATE INDEX idx_sensor_created ON sensor(created DESC);
+
+-- Trigger
+CREATE TRIGGER update_sensor_updated
+    BEFORE UPDATE ON sensor
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE sensor IS 'Sensors monitor for events and create trigger instances';
+COMMENT ON COLUMN sensor.ref IS 'Unique sensor reference (format: pack.name)';
+COMMENT ON COLUMN sensor.label IS 'Human-readable sensor name';
+COMMENT ON COLUMN sensor.entrypoint IS 'Script or command to execute';
+COMMENT ON COLUMN sensor.runtime IS 'Runtime environment for execution';
+COMMENT ON COLUMN sensor.trigger IS 'Trigger type this sensor creates events for';
+COMMENT ON COLUMN sensor.enabled IS 'Whether this sensor is active';
+COMMENT ON COLUMN sensor.is_adhoc IS 'True if sensor was manually created (ad-hoc), false if installed from pack';
+COMMENT ON COLUMN sensor.runtime_version_constraint IS 'Semver version constraint for the runtime (e.g., ">=3.12", ">=3.12,<4.0", "~18.0"). NULL means any version.';
+
+-- ============================================================================
+-- EVENT TABLE
+-- ============================================================================
+
+CREATE TABLE event (
+    id BIGSERIAL PRIMARY KEY,
+    trigger BIGINT REFERENCES trigger(id) ON DELETE SET NULL,
+    trigger_ref TEXT NOT NULL,
+    config JSONB,
+    payload JSONB,
+    source BIGINT REFERENCES sensor(id) ON DELETE SET NULL,
+    source_ref TEXT,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    rule BIGINT,
+    rule_ref TEXT
+);
+
+-- Indexes
+CREATE INDEX idx_event_trigger ON event(trigger);
+CREATE INDEX idx_event_trigger_ref ON event(trigger_ref);
+CREATE INDEX idx_event_source ON event(source);
+CREATE INDEX idx_event_created ON event(created DESC);
+CREATE INDEX idx_event_trigger_created ON event(trigger, created DESC);
+CREATE INDEX idx_event_trigger_ref_created ON event(trigger_ref, created DESC);
+CREATE INDEX idx_event_source_created ON event(source, created DESC);
+CREATE INDEX idx_event_payload_gin ON event USING GIN (payload);
+
+-- Comments
+COMMENT ON TABLE event IS 'Events are instances of triggers firing';
+COMMENT ON COLUMN event.trigger IS 'Trigger that fired (may be null if trigger deleted)';
+COMMENT ON COLUMN event.trigger_ref IS 'Trigger reference (preserved even if trigger deleted)';
+COMMENT ON COLUMN event.config IS 'Snapshot of trigger/sensor configuration at event time';
+COMMENT ON COLUMN event.payload IS 'Event data payload';
+COMMENT ON COLUMN event.source IS 'Sensor that generated this event';
+
+-- ============================================================================
+-- ENFORCEMENT TABLE
+-- ============================================================================
+
+CREATE TABLE enforcement (
+    id BIGSERIAL PRIMARY KEY,
+    rule BIGINT, -- Forward reference to rule table, will add constraint after rule is created
+    rule_ref TEXT NOT NULL,
+    trigger_ref TEXT NOT NULL,
+    config JSONB,
+    event BIGINT,           -- references event(id); no FK because event becomes a hypertable
+    status enforcement_status_enum NOT NULL DEFAULT 'created',
+    payload JSONB NOT NULL,
+    condition enforcement_condition_enum NOT NULL DEFAULT 'all',
+    conditions JSONB NOT NULL DEFAULT '[]'::jsonb,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    resolved_at TIMESTAMPTZ,
+
+    -- Constraints
+    CONSTRAINT enforcement_condition_check CHECK (condition IN ('any', 'all'))
+);
+
+-- Indexes
+CREATE INDEX idx_enforcement_rule ON enforcement(rule);
+CREATE INDEX idx_enforcement_rule_ref ON enforcement(rule_ref);
+CREATE INDEX idx_enforcement_trigger_ref ON enforcement(trigger_ref);
+CREATE INDEX idx_enforcement_event ON enforcement(event);
+CREATE INDEX idx_enforcement_status ON enforcement(status);
+CREATE INDEX idx_enforcement_created ON enforcement(created DESC);
+CREATE INDEX idx_enforcement_status_created ON enforcement(status, created DESC);
+CREATE INDEX idx_enforcement_rule_status ON enforcement(rule, status);
+CREATE INDEX idx_enforcement_event_status ON enforcement(event, status);
+CREATE INDEX idx_enforcement_payload_gin ON enforcement USING GIN (payload);
+CREATE INDEX idx_enforcement_conditions_gin ON enforcement USING GIN (conditions);
+
+-- Comments
+COMMENT ON TABLE enforcement IS 'Enforcements represent rule triggering by events';
+COMMENT ON COLUMN enforcement.rule IS 'Rule being enforced (may be null if rule deleted)';
+COMMENT ON COLUMN enforcement.rule_ref IS 'Rule reference (preserved even if rule deleted)';
+COMMENT ON COLUMN enforcement.event IS 'Event that triggered this enforcement (no FK — event is a hypertable)';
+COMMENT ON COLUMN enforcement.status IS 'Processing status (created → processed or disabled)';
+COMMENT ON COLUMN enforcement.resolved_at IS 'Timestamp when the enforcement was resolved (status changed from created to processed/disabled). NULL while status is created.';
+COMMENT ON COLUMN enforcement.payload IS 'Event payload for rule evaluation';
+COMMENT ON COLUMN enforcement.condition IS 'Logical operator for conditions (any=OR, all=AND)';
+COMMENT ON COLUMN enforcement.conditions IS 'Condition expressions to evaluate';
+
+-- ============================================================================
+-- ACTION TABLE
+-- ============================================================================
+
+CREATE TABLE action (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT NOT NULL,
+    label TEXT NOT NULL,
+    description TEXT,
+    entrypoint TEXT NOT NULL,
+    runtime BIGINT REFERENCES runtime(id),
+    param_schema JSONB,
+    out_schema JSONB,
+    parameter_delivery TEXT NOT NULL DEFAULT 'stdin' CHECK (parameter_delivery IN ('stdin', 'file')),
+    parameter_format TEXT NOT NULL DEFAULT 'json' CHECK (parameter_format IN ('dotenv', 'json', 'yaml')),
+    output_format TEXT NOT NULL DEFAULT 'text' CHECK (output_format IN ('text', 'json', 'yaml', 'jsonl')),
+    is_adhoc BOOLEAN NOT NULL DEFAULT FALSE,
+    timeout_seconds INTEGER,
+    max_retries INTEGER DEFAULT 0,
+    runtime_version_constraint TEXT,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT action_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT action_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$')
+);
+
+-- Indexes
+CREATE INDEX idx_action_ref ON action(ref);
+CREATE INDEX idx_action_pack ON action(pack);
+CREATE INDEX idx_action_runtime ON action(runtime);
+CREATE INDEX idx_action_parameter_delivery ON action(parameter_delivery);
+CREATE INDEX idx_action_parameter_format ON action(parameter_format);
+CREATE INDEX idx_action_output_format ON action(output_format);
+CREATE INDEX idx_action_is_adhoc ON action(is_adhoc) WHERE is_adhoc = true;
+CREATE INDEX idx_action_created ON action(created DESC);
+
+-- Trigger
+CREATE TRIGGER update_action_updated
+    BEFORE UPDATE ON action
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE action IS 'Actions are executable tasks that can be triggered';
+COMMENT ON COLUMN action.ref IS 'Unique action reference (format: pack.name)';
+COMMENT ON COLUMN action.pack IS 'Pack this action belongs to';
+COMMENT ON COLUMN action.label IS 'Human-readable action name';
+COMMENT ON COLUMN action.entrypoint IS 'Script or command to execute';
+COMMENT ON COLUMN action.runtime IS 'Runtime environment for execution';
+COMMENT ON COLUMN action.param_schema IS 'JSON schema for action parameters';
+COMMENT ON COLUMN action.out_schema IS 'JSON schema for action output';
+COMMENT ON COLUMN action.parameter_delivery IS 'How parameters are delivered: stdin (standard input - secure), file (temporary file - secure for large payloads). Environment variables are set separately via execution.env_vars.';
+COMMENT ON COLUMN action.parameter_format IS 'Parameter serialization format: json (JSON object - default), dotenv (KEY=''VALUE''), yaml (YAML format)';
+COMMENT ON COLUMN action.output_format IS 'Output parsing format: text (no parsing - raw stdout), json (parse stdout as JSON), yaml (parse stdout as YAML), jsonl (parse each line as JSON, collect into array)';
+COMMENT ON COLUMN action.is_adhoc IS 'True if action was manually created (ad-hoc), false if installed from pack';
+COMMENT ON COLUMN action.timeout_seconds IS 'Worker queue TTL override in seconds. If NULL, uses global worker_queue_ttl_ms config. Allows per-action timeout tuning.';
+COMMENT ON COLUMN action.max_retries IS 'Maximum number of automatic retry attempts for failed executions. 0 = no retries (default).';
+COMMENT ON COLUMN action.runtime_version_constraint IS 'Semver version constraint for the runtime (e.g., ">=3.12", ">=3.12,<4.0", "~18.0"). NULL means any version.';
+
+-- ============================================================================
+
+-- Add foreign key constraint for policy table
+ALTER TABLE policy
+    ADD CONSTRAINT policy_action_fkey
+    FOREIGN KEY (action) REFERENCES action(id) ON DELETE CASCADE;
+
+-- Note: Foreign key constraints for key table (key_owner_action_fkey, key_owner_sensor_fkey)
+-- will be added in migration 000007_supporting_systems.sql after the key table is created
+
+-- Note: Rule table will be created in migration 000005 after execution table exists
+-- Note: Foreign key constraints for enforcement.rule and event.rule will be added there

docker/distributable/migrations/20250101000005_execution_and_operations.sql

@@ -0,0 +1,410 @@
+-- Migration: Execution and Operations
+-- Description: Creates execution, inquiry, rule, worker, and notification tables.
+--              Includes retry tracking, worker health views, and helper functions.
+--              Consolidates former migrations: 000006 (execution_system), 000008
+--              (worker_notification), 000014 (worker_table), and 20260209 (phase3).
+--
+--              NOTE: The execution table is converted to a TimescaleDB hypertable in
+--              migration 000009. Hypertables cannot be the target of FK constraints,
+--              so columns referencing execution (inquiry.execution, workflow_execution.execution)
+--              are plain BIGINT with no FK. Similarly, columns ON the execution table that
+--              would self-reference or reference other hypertables (parent, enforcement,
+--              original_execution) are plain BIGINT. The action and executor FKs are also
+--              omitted since they would need to be dropped during hypertable conversion.
+-- Version: 20250101000005
+
+-- ============================================================================
+-- EXECUTION TABLE
+-- ============================================================================
+
+CREATE TABLE execution (
+    id BIGSERIAL PRIMARY KEY,
+    action BIGINT,          -- references action(id); no FK because execution becomes a hypertable
+    action_ref TEXT NOT NULL,
+    config JSONB,
+    env_vars JSONB,
+    parent BIGINT,          -- self-reference; no FK because execution becomes a hypertable
+    enforcement BIGINT,     -- references enforcement(id); no FK (both are hypertables)
+    executor BIGINT,        -- references identity(id); no FK because execution becomes a hypertable
+    worker BIGINT,          -- references worker(id); no FK because execution becomes a hypertable
+    status execution_status_enum NOT NULL DEFAULT 'requested',
+    result JSONB,
+    started_at TIMESTAMPTZ,         -- set when execution transitions to 'running'
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    is_workflow BOOLEAN DEFAULT false NOT NULL,
+    workflow_def BIGINT,    -- references workflow_definition(id); no FK because execution becomes a hypertable
+    workflow_task JSONB,
+
+    -- Retry tracking (baked in from phase 3)
+    retry_count INTEGER NOT NULL DEFAULT 0,
+    max_retries INTEGER,
+    retry_reason TEXT,
+    original_execution BIGINT, -- self-reference; no FK because execution becomes a hypertable
+
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_execution_action ON execution(action);
+CREATE INDEX idx_execution_action_ref ON execution(action_ref);
+CREATE INDEX idx_execution_parent ON execution(parent);
+CREATE INDEX idx_execution_enforcement ON execution(enforcement);
+CREATE INDEX idx_execution_executor ON execution(executor);
+CREATE INDEX idx_execution_worker ON execution(worker);
+CREATE INDEX idx_execution_status ON execution(status);
+CREATE INDEX idx_execution_created ON execution(created DESC);
+CREATE INDEX idx_execution_updated ON execution(updated DESC);
+CREATE INDEX idx_execution_status_created ON execution(status, created DESC);
+CREATE INDEX idx_execution_status_updated ON execution(status, updated DESC);
+CREATE INDEX idx_execution_action_status ON execution(action, status);
+CREATE INDEX idx_execution_executor_created ON execution(executor, created DESC);
+CREATE INDEX idx_execution_worker_created ON execution(worker, created DESC);
+CREATE INDEX idx_execution_parent_created ON execution(parent, created DESC);
+CREATE INDEX idx_execution_result_gin ON execution USING GIN (result);
+CREATE INDEX idx_execution_env_vars_gin ON execution USING GIN (env_vars);
+CREATE INDEX idx_execution_original_execution ON execution(original_execution) WHERE original_execution IS NOT NULL;
+CREATE INDEX idx_execution_status_retry ON execution(status, retry_count) WHERE status = 'failed' AND retry_count < COALESCE(max_retries, 0);
+
+-- Trigger
+CREATE TRIGGER update_execution_updated
+    BEFORE UPDATE ON execution
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE execution IS 'Executions represent action runs, supports nested workflows';
+COMMENT ON COLUMN execution.action IS 'Action being executed (may be null if action deleted)';
+COMMENT ON COLUMN execution.action_ref IS 'Action reference (preserved even if action deleted)';
+COMMENT ON COLUMN execution.config IS 'Snapshot of action configuration at execution time';
+COMMENT ON COLUMN execution.env_vars IS 'Environment variables for this execution as key-value pairs (string -> string). These are set in the execution environment and are separate from action parameters. Used for execution context, configuration, and non-sensitive metadata.';
+COMMENT ON COLUMN execution.parent IS 'Parent execution ID for workflow hierarchies (no FK — execution is a hypertable)';
+COMMENT ON COLUMN execution.enforcement IS 'Enforcement that triggered this execution (no FK — both are hypertables)';
+COMMENT ON COLUMN execution.executor IS 'Identity that initiated the execution (no FK — execution is a hypertable)';
+COMMENT ON COLUMN execution.worker IS 'Assigned worker handling this execution (no FK — execution is a hypertable)';
+COMMENT ON COLUMN execution.status IS 'Current execution lifecycle status';
+COMMENT ON COLUMN execution.result IS 'Execution output/results';
+COMMENT ON COLUMN execution.retry_count IS 'Current retry attempt number (0 = first attempt, 1 = first retry, etc.)';
+COMMENT ON COLUMN execution.max_retries IS 'Maximum retries for this execution. Copied from action.max_retries at creation time.';
+COMMENT ON COLUMN execution.retry_reason IS 'Reason for retry (e.g., "worker_unavailable", "transient_error", "manual_retry")';
+COMMENT ON COLUMN execution.original_execution IS 'ID of the original execution if this is a retry. Forms a retry chain.';
+
+-- ============================================================================
+
+-- ============================================================================
+-- INQUIRY TABLE
+-- ============================================================================
+
+CREATE TABLE inquiry (
+    id BIGSERIAL PRIMARY KEY,
+    execution BIGINT NOT NULL, -- references execution(id); no FK because execution is a hypertable
+    prompt TEXT NOT NULL,
+    response_schema JSONB,
+    assigned_to BIGINT REFERENCES identity(id) ON DELETE SET NULL,
+    status inquiry_status_enum NOT NULL DEFAULT 'pending',
+    response JSONB,
+    timeout_at TIMESTAMPTZ,
+    responded_at TIMESTAMPTZ,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_inquiry_execution ON inquiry(execution);
+CREATE INDEX idx_inquiry_assigned_to ON inquiry(assigned_to);
+CREATE INDEX idx_inquiry_status ON inquiry(status);
+CREATE INDEX idx_inquiry_timeout_at ON inquiry(timeout_at) WHERE timeout_at IS NOT NULL;
+CREATE INDEX idx_inquiry_created ON inquiry(created DESC);
+CREATE INDEX idx_inquiry_status_created ON inquiry(status, created DESC);
+CREATE INDEX idx_inquiry_assigned_status ON inquiry(assigned_to, status);
+CREATE INDEX idx_inquiry_execution_status ON inquiry(execution, status);
+CREATE INDEX idx_inquiry_response_gin ON inquiry USING GIN (response);
+
+-- Trigger
+CREATE TRIGGER update_inquiry_updated
+    BEFORE UPDATE ON inquiry
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE inquiry IS 'Inquiries enable human-in-the-loop workflows with async user interactions';
+COMMENT ON COLUMN inquiry.execution IS 'Execution that is waiting on this inquiry (no FK — execution is a hypertable)';
+COMMENT ON COLUMN inquiry.prompt IS 'Question or prompt text for the user';
+COMMENT ON COLUMN inquiry.response_schema IS 'JSON schema defining expected response format';
+COMMENT ON COLUMN inquiry.assigned_to IS 'Identity who should respond to this inquiry';
+COMMENT ON COLUMN inquiry.status IS 'Current inquiry lifecycle status';
+COMMENT ON COLUMN inquiry.response IS 'User response data';
+COMMENT ON COLUMN inquiry.timeout_at IS 'When this inquiry expires';
+COMMENT ON COLUMN inquiry.responded_at IS 'When the response was received';
+
+-- ============================================================================
+
+-- ============================================================================
+-- RULE TABLE
+-- ============================================================================
+
+CREATE TABLE rule (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    pack BIGINT NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT NOT NULL,
+    label TEXT NOT NULL,
+    description TEXT,
+    action BIGINT REFERENCES action(id) ON DELETE SET NULL,
+    action_ref TEXT NOT NULL,
+    trigger BIGINT REFERENCES trigger(id) ON DELETE SET NULL,
+    trigger_ref TEXT NOT NULL,
+    conditions JSONB NOT NULL DEFAULT '[]'::jsonb,
+    action_params JSONB DEFAULT '{}'::jsonb,
+    trigger_params JSONB DEFAULT '{}'::jsonb,
+    enabled BOOLEAN NOT NULL,
+    is_adhoc BOOLEAN NOT NULL DEFAULT FALSE,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT rule_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT rule_ref_format CHECK (ref ~ '^[^.]+\.[^.]+$')
+);
+
+-- Indexes
+CREATE INDEX idx_rule_ref ON rule(ref);
+CREATE INDEX idx_rule_pack ON rule(pack);
+CREATE INDEX idx_rule_action ON rule(action);
+CREATE INDEX idx_rule_trigger ON rule(trigger);
+CREATE INDEX idx_rule_enabled ON rule(enabled) WHERE enabled = TRUE;
+CREATE INDEX idx_rule_is_adhoc ON rule(is_adhoc) WHERE is_adhoc = true;
+CREATE INDEX idx_rule_created ON rule(created DESC);
+CREATE INDEX idx_rule_trigger_enabled ON rule(trigger, enabled);
+CREATE INDEX idx_rule_action_enabled ON rule(action, enabled);
+CREATE INDEX idx_rule_pack_enabled ON rule(pack, enabled);
+CREATE INDEX idx_rule_action_params_gin ON rule USING GIN (action_params);
+CREATE INDEX idx_rule_trigger_params_gin ON rule USING GIN (trigger_params);
+
+-- Trigger
+CREATE TRIGGER update_rule_updated
+    BEFORE UPDATE ON rule
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE rule IS 'Rules link triggers to actions with conditions';
+COMMENT ON COLUMN rule.ref IS 'Unique rule reference (format: pack.name)';
+COMMENT ON COLUMN rule.label IS 'Human-readable rule name';
+COMMENT ON COLUMN rule.action IS 'Action to execute when rule triggers (null if action deleted)';
+COMMENT ON COLUMN rule.trigger IS 'Trigger that activates this rule (null if trigger deleted)';
+COMMENT ON COLUMN rule.conditions IS 'Condition expressions to evaluate before executing action';
+COMMENT ON COLUMN rule.action_params IS 'Parameter overrides for the action';
+COMMENT ON COLUMN rule.trigger_params IS 'Parameter overrides for the trigger';
+COMMENT ON COLUMN rule.enabled IS 'Whether this rule is active';
+COMMENT ON COLUMN rule.is_adhoc IS 'True if rule was manually created (ad-hoc), false if installed from pack';
+
+-- ============================================================================
+
+-- Add foreign key constraints now that rule table exists
+ALTER TABLE enforcement
+    ADD CONSTRAINT enforcement_rule_fkey
+    FOREIGN KEY (rule) REFERENCES rule(id) ON DELETE SET NULL;
+
+ALTER TABLE event
+    ADD CONSTRAINT event_rule_fkey
+    FOREIGN KEY (rule) REFERENCES rule(id) ON DELETE SET NULL;
+
+-- ============================================================================
+-- WORKER TABLE
+-- ============================================================================
+
+CREATE TABLE worker (
+    id BIGSERIAL PRIMARY KEY,
+    name TEXT NOT NULL UNIQUE,
+    worker_type worker_type_enum NOT NULL,
+    worker_role worker_role_enum NOT NULL,
+    runtime BIGINT REFERENCES runtime(id) ON DELETE SET NULL,
+    host TEXT,
+    port INTEGER,
+    status worker_status_enum NOT NULL DEFAULT 'active',
+    capabilities JSONB,
+    meta JSONB,
+    last_heartbeat TIMESTAMPTZ,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_worker_name ON worker(name);
+CREATE INDEX idx_worker_type ON worker(worker_type);
+CREATE INDEX idx_worker_role ON worker(worker_role);
+CREATE INDEX idx_worker_runtime ON worker(runtime);
+CREATE INDEX idx_worker_status ON worker(status);
+CREATE INDEX idx_worker_last_heartbeat ON worker(last_heartbeat DESC) WHERE last_heartbeat IS NOT NULL;
+CREATE INDEX idx_worker_created ON worker(created DESC);
+CREATE INDEX idx_worker_status_role ON worker(status, worker_role);
+CREATE INDEX idx_worker_capabilities_gin ON worker USING GIN (capabilities);
+CREATE INDEX idx_worker_meta_gin ON worker USING GIN (meta);
+CREATE INDEX idx_worker_capabilities_health_status ON worker USING GIN ((capabilities -> 'health' -> 'status'));
+
+-- Trigger
+CREATE TRIGGER update_worker_updated
+    BEFORE UPDATE ON worker
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE worker IS 'Worker registration and tracking table for action and sensor workers';
+COMMENT ON COLUMN worker.name IS 'Unique worker identifier (typically hostname-based)';
+COMMENT ON COLUMN worker.worker_type IS 'Worker deployment type (local or remote)';
+COMMENT ON COLUMN worker.worker_role IS 'Worker role (action or sensor)';
+COMMENT ON COLUMN worker.runtime IS 'Runtime environment this worker supports (optional)';
+COMMENT ON COLUMN worker.host IS 'Worker host address';
+COMMENT ON COLUMN worker.port IS 'Worker port number';
+COMMENT ON COLUMN worker.status IS 'Worker operational status';
+COMMENT ON COLUMN worker.capabilities IS 'Worker capabilities (e.g., max_concurrent_executions, supported runtimes)';
+COMMENT ON COLUMN worker.meta IS 'Additional worker metadata';
+COMMENT ON COLUMN worker.last_heartbeat IS 'Timestamp of last heartbeat from worker';
+
+-- ============================================================================
+-- NOTIFICATION TABLE
+-- ============================================================================
+
+CREATE TABLE notification (
+    id BIGSERIAL PRIMARY KEY,
+    channel TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity TEXT NOT NULL,
+    activity TEXT NOT NULL,
+    state notification_status_enum NOT NULL DEFAULT 'created',
+    content JSONB,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_notification_channel ON notification(channel);
+CREATE INDEX idx_notification_entity_type ON notification(entity_type);
+CREATE INDEX idx_notification_entity ON notification(entity);
+CREATE INDEX idx_notification_state ON notification(state);
+CREATE INDEX idx_notification_created ON notification(created DESC);
+CREATE INDEX idx_notification_channel_state ON notification(channel, state);
+CREATE INDEX idx_notification_entity_type_entity ON notification(entity_type, entity);
+CREATE INDEX idx_notification_state_created ON notification(state, created DESC);
+CREATE INDEX idx_notification_content_gin ON notification USING GIN (content);
+
+-- Trigger
+CREATE TRIGGER update_notification_updated
+    BEFORE UPDATE ON notification
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Function for pg_notify on notification insert
+CREATE OR REPLACE FUNCTION notify_on_insert()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload TEXT;
+BEGIN
+    -- Build JSON payload with id, entity, and activity
+    payload := json_build_object(
+        'id', NEW.id,
+        'entity_type', NEW.entity_type,
+        'entity', NEW.entity,
+        'activity', NEW.activity
+    )::text;
+
+    -- Send notification to the specified channel
+    PERFORM pg_notify(NEW.channel, payload);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger to send pg_notify on notification insert
+CREATE TRIGGER notify_on_notification_insert
+    AFTER INSERT ON notification
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_on_insert();
+
+-- Comments
+COMMENT ON TABLE notification IS 'System notifications about entity changes for real-time updates';
+COMMENT ON COLUMN notification.channel IS 'Notification channel (typically table name)';
+COMMENT ON COLUMN notification.entity_type IS 'Type of entity (table name)';
+COMMENT ON COLUMN notification.entity IS 'Entity identifier (typically ID or ref)';
+COMMENT ON COLUMN notification.activity IS 'Activity type (e.g., "created", "updated", "completed")';
+COMMENT ON COLUMN notification.state IS 'Processing state of notification';
+COMMENT ON COLUMN notification.content IS 'Optional notification payload data';
+
+-- ============================================================================
+-- WORKER HEALTH VIEWS AND FUNCTIONS
+-- ============================================================================
+
+-- View for healthy workers (convenience for queries)
+CREATE OR REPLACE VIEW healthy_workers AS
+SELECT
+    w.id,
+    w.name,
+    w.worker_type,
+    w.worker_role,
+    w.runtime,
+    w.status,
+    w.capabilities,
+    w.last_heartbeat,
+    (w.capabilities -> 'health' ->> 'status')::TEXT as health_status,
+    (w.capabilities -> 'health' ->> 'queue_depth')::INTEGER as queue_depth,
+    (w.capabilities -> 'health' ->> 'consecutive_failures')::INTEGER as consecutive_failures
+FROM worker w
+WHERE
+    w.status = 'active'
+    AND w.last_heartbeat > NOW() - INTERVAL '30 seconds'
+    AND (
+        -- Healthy if no health info (backward compatible)
+        w.capabilities -> 'health' IS NULL
+        OR
+        -- Or explicitly marked healthy
+        w.capabilities -> 'health' ->> 'status' IN ('healthy', 'degraded')
+    );
+
+COMMENT ON VIEW healthy_workers IS 'Workers that are active, have fresh heartbeat, and are healthy or degraded (not unhealthy)';
+
+-- Function to get worker queue depth estimate
+CREATE OR REPLACE FUNCTION get_worker_queue_depth(worker_id_param BIGINT)
+RETURNS INTEGER AS $$
+BEGIN
+    RETURN (
+        SELECT (capabilities -> 'health' ->> 'queue_depth')::INTEGER
+        FROM worker
+        WHERE id = worker_id_param
+    );
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION get_worker_queue_depth IS 'Extract current queue depth from worker health metadata';
+
+-- Function to check if execution is retriable
+CREATE OR REPLACE FUNCTION is_execution_retriable(execution_id_param BIGINT)
+RETURNS BOOLEAN AS $$
+DECLARE
+    exec_record RECORD;
+BEGIN
+    SELECT
+        e.retry_count,
+        e.max_retries,
+        e.status
+    INTO exec_record
+    FROM execution e
+    WHERE e.id = execution_id_param;
+
+    IF NOT FOUND THEN
+        RETURN FALSE;
+    END IF;
+
+    -- Can retry if:
+    -- 1. Status is failed
+    -- 2. max_retries is set and > 0
+    -- 3. retry_count < max_retries
+    RETURN (
+        exec_record.status = 'failed'
+        AND exec_record.max_retries IS NOT NULL
+        AND exec_record.max_retries > 0
+        AND exec_record.retry_count < exec_record.max_retries
+    );
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION is_execution_retriable IS 'Check if a failed execution can be automatically retried based on retry limits';

docker/distributable/migrations/20250101000006_workflow_system.sql

@@ -0,0 +1,145 @@
+-- Migration: Workflow System
+-- Description: Creates workflow_definition and workflow_execution tables
+--              (workflow_task_execution consolidated into execution.workflow_task JSONB)
+--
+--              NOTE: The execution table is converted to a TimescaleDB hypertable in
+--              migration 000009. Hypertables cannot be the target of FK constraints,
+--              so workflow_execution.execution is a plain BIGINT with no FK.
+--              execution.workflow_def also has no FK (added as plain BIGINT in 000005)
+--              since execution is a hypertable and FKs from hypertables are only
+--              supported for simple cases — we omit it for consistency.
+-- Version: 20250101000006
+
+-- ============================================================================
+-- WORKFLOW DEFINITION TABLE
+-- ============================================================================
+
+CREATE TABLE workflow_definition (
+    id BIGSERIAL PRIMARY KEY,
+    ref VARCHAR(255) NOT NULL UNIQUE,
+    pack BIGINT NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref VARCHAR(255) NOT NULL,
+    label VARCHAR(255) NOT NULL,
+    description TEXT,
+    version VARCHAR(50) NOT NULL,
+    param_schema JSONB,
+    out_schema JSONB,
+    definition JSONB NOT NULL,
+    tags TEXT[] DEFAULT '{}',
+    created TIMESTAMPTZ DEFAULT NOW() NOT NULL,
+    updated TIMESTAMPTZ DEFAULT NOW() NOT NULL
+);
+
+-- Indexes
+CREATE INDEX idx_workflow_def_pack ON workflow_definition(pack);
+CREATE INDEX idx_workflow_def_ref ON workflow_definition(ref);
+CREATE INDEX idx_workflow_def_tags ON workflow_definition USING gin(tags);
+
+-- Trigger
+CREATE TRIGGER update_workflow_definition_updated
+    BEFORE UPDATE ON workflow_definition
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE workflow_definition IS 'Stores workflow definitions (YAML parsed to JSON)';
+COMMENT ON COLUMN workflow_definition.ref IS 'Unique workflow reference (e.g., pack_name.workflow_name)';
+COMMENT ON COLUMN workflow_definition.definition IS 'Complete workflow specification including tasks, variables, and transitions';
+COMMENT ON COLUMN workflow_definition.param_schema IS 'JSON schema for workflow input parameters';
+COMMENT ON COLUMN workflow_definition.out_schema IS 'JSON schema for workflow output';
+
+-- ============================================================================
+-- WORKFLOW EXECUTION TABLE
+-- ============================================================================
+
+CREATE TABLE workflow_execution (
+    id BIGSERIAL PRIMARY KEY,
+    execution BIGINT NOT NULL, -- references execution(id); no FK because execution is a hypertable
+    workflow_def BIGINT NOT NULL REFERENCES workflow_definition(id) ON DELETE CASCADE,
+    current_tasks TEXT[] DEFAULT '{}',
+    completed_tasks TEXT[] DEFAULT '{}',
+    failed_tasks TEXT[] DEFAULT '{}',
+    skipped_tasks TEXT[] DEFAULT '{}',
+    variables JSONB DEFAULT '{}',
+    task_graph JSONB NOT NULL,
+    status execution_status_enum NOT NULL DEFAULT 'requested',
+    error_message TEXT,
+    paused BOOLEAN DEFAULT false NOT NULL,
+    pause_reason TEXT,
+    created TIMESTAMPTZ DEFAULT NOW() NOT NULL,
+    updated TIMESTAMPTZ DEFAULT NOW() NOT NULL
+);
+
+-- Indexes
+CREATE INDEX idx_workflow_exec_execution ON workflow_execution(execution);
+CREATE INDEX idx_workflow_exec_workflow_def ON workflow_execution(workflow_def);
+CREATE INDEX idx_workflow_exec_status ON workflow_execution(status);
+CREATE INDEX idx_workflow_exec_paused ON workflow_execution(paused) WHERE paused = true;
+
+-- Trigger
+CREATE TRIGGER update_workflow_execution_updated
+    BEFORE UPDATE ON workflow_execution
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE workflow_execution IS 'Runtime state tracking for workflow executions. execution column has no FK — execution is a hypertable.';
+COMMENT ON COLUMN workflow_execution.variables IS 'Workflow-scoped variables, updated via publish directives';
+COMMENT ON COLUMN workflow_execution.task_graph IS 'Execution graph with dependencies and transitions';
+COMMENT ON COLUMN workflow_execution.current_tasks IS 'Array of task names currently executing';
+COMMENT ON COLUMN workflow_execution.paused IS 'True if workflow execution is paused (can be resumed)';
+
+-- ============================================================================
+-- MODIFY ACTION TABLE - Add Workflow Support
+-- ============================================================================
+
+ALTER TABLE action
+    ADD COLUMN workflow_def BIGINT REFERENCES workflow_definition(id) ON DELETE CASCADE;
+
+CREATE INDEX idx_action_workflow_def ON action(workflow_def);
+
+COMMENT ON COLUMN action.workflow_def IS 'Reference to workflow definition (non-null means this action is a workflow)';
+
+-- NOTE: execution.workflow_def has no FK constraint because execution is a
+-- TimescaleDB hypertable (converted in migration 000009). The column was
+-- created as a plain BIGINT in migration 000005.
+
+-- ============================================================================
+-- WORKFLOW VIEWS
+-- ============================================================================
+
+CREATE VIEW workflow_execution_summary AS
+SELECT
+    we.id,
+    we.execution,
+    wd.ref as workflow_ref,
+    wd.label as workflow_label,
+    wd.version as workflow_version,
+    we.status,
+    we.paused,
+    array_length(we.current_tasks, 1) as current_task_count,
+    array_length(we.completed_tasks, 1) as completed_task_count,
+    array_length(we.failed_tasks, 1) as failed_task_count,
+    array_length(we.skipped_tasks, 1) as skipped_task_count,
+    we.error_message,
+    we.created,
+    we.updated
+FROM workflow_execution we
+JOIN workflow_definition wd ON we.workflow_def = wd.id;
+
+COMMENT ON VIEW workflow_execution_summary IS 'Summary view of workflow executions with task counts';
+
+CREATE VIEW workflow_action_link AS
+SELECT
+    wd.id as workflow_def_id,
+    wd.ref as workflow_ref,
+    wd.label,
+    wd.version,
+    a.id as action_id,
+    a.ref as action_ref,
+    a.pack as pack_id,
+    a.pack_ref
+FROM workflow_definition wd
+LEFT JOIN action a ON a.workflow_def = wd.id;
+
+COMMENT ON VIEW workflow_action_link IS 'Links workflow definitions to their corresponding action records';

docker/distributable/migrations/20250101000007_supporting_systems.sql

@@ -0,0 +1,779 @@
+-- Migration: Supporting Systems
+-- Description: Creates keys, artifacts, queue_stats, pack_environment, pack_testing,
+--              and webhook function tables.
+--              Consolidates former migrations: 000009 (keys_artifacts), 000010 (webhook_system),
+--              000011 (pack_environments), and 000012 (pack_testing).
+-- Version: 20250101000007
+
+-- ============================================================================
+-- KEY TABLE
+-- ============================================================================
+
+CREATE TABLE key (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL UNIQUE,
+    owner_type owner_type_enum NOT NULL,
+    owner TEXT,
+    owner_identity BIGINT REFERENCES identity(id),
+    owner_pack BIGINT REFERENCES pack(id),
+    owner_pack_ref TEXT,
+    owner_action BIGINT, -- Forward reference to action table
+    owner_action_ref TEXT,
+    owner_sensor BIGINT, -- Forward reference to sensor table
+    owner_sensor_ref TEXT,
+    name TEXT NOT NULL,
+    encrypted BOOLEAN NOT NULL,
+    encryption_key_hash TEXT,
+    value TEXT NOT NULL,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Constraints
+    CONSTRAINT key_ref_lowercase CHECK (ref = LOWER(ref)),
+    CONSTRAINT key_ref_format CHECK (ref ~ '^[^.]+(\.[^.]+)*$')
+);
+
+-- Unique index on owner_type, owner, name
+CREATE UNIQUE INDEX idx_key_unique ON key(owner_type, owner, name);
+
+-- Indexes
+CREATE INDEX idx_key_ref ON key(ref);
+CREATE INDEX idx_key_owner_type ON key(owner_type);
+CREATE INDEX idx_key_owner_identity ON key(owner_identity);
+CREATE INDEX idx_key_owner_pack ON key(owner_pack);
+CREATE INDEX idx_key_owner_action ON key(owner_action);
+CREATE INDEX idx_key_owner_sensor ON key(owner_sensor);
+CREATE INDEX idx_key_created ON key(created DESC);
+CREATE INDEX idx_key_owner_type_owner ON key(owner_type, owner);
+CREATE INDEX idx_key_owner_identity_name ON key(owner_identity, name);
+CREATE INDEX idx_key_owner_pack_name ON key(owner_pack, name);
+
+-- Function to validate and set owner fields
+CREATE OR REPLACE FUNCTION validate_key_owner()
+RETURNS TRIGGER AS $$
+DECLARE
+    owner_count INTEGER := 0;
+BEGIN
+    -- Count how many owner fields are set
+    IF NEW.owner_identity IS NOT NULL THEN owner_count := owner_count + 1; END IF;
+    IF NEW.owner_pack IS NOT NULL THEN owner_count := owner_count + 1; END IF;
+    IF NEW.owner_action IS NOT NULL THEN owner_count := owner_count + 1; END IF;
+    IF NEW.owner_sensor IS NOT NULL THEN owner_count := owner_count + 1; END IF;
+
+    -- System owner should have no owner fields set
+    IF NEW.owner_type = 'system' THEN
+        IF owner_count > 0 THEN
+            RAISE EXCEPTION 'System owner cannot have specific owner fields set';
+        END IF;
+        NEW.owner := 'system';
+    -- All other types must have exactly one owner field set
+    ELSIF owner_count != 1 THEN
+        RAISE EXCEPTION 'Exactly one owner field must be set for owner_type %', NEW.owner_type;
+    -- Validate owner_type matches the populated field and set owner
+    ELSIF NEW.owner_type = 'identity' THEN
+        IF NEW.owner_identity IS NULL THEN
+            RAISE EXCEPTION 'owner_identity must be set for owner_type identity';
+        END IF;
+        NEW.owner := NEW.owner_identity::TEXT;
+    ELSIF NEW.owner_type = 'pack' THEN
+        IF NEW.owner_pack IS NULL THEN
+            RAISE EXCEPTION 'owner_pack must be set for owner_type pack';
+        END IF;
+        NEW.owner := NEW.owner_pack::TEXT;
+    ELSIF NEW.owner_type = 'action' THEN
+        IF NEW.owner_action IS NULL THEN
+            RAISE EXCEPTION 'owner_action must be set for owner_type action';
+        END IF;
+        NEW.owner := NEW.owner_action::TEXT;
+    ELSIF NEW.owner_type = 'sensor' THEN
+        IF NEW.owner_sensor IS NULL THEN
+            RAISE EXCEPTION 'owner_sensor must be set for owner_type sensor';
+        END IF;
+        NEW.owner := NEW.owner_sensor::TEXT;
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger to validate owner fields
+CREATE TRIGGER validate_key_owner_trigger
+    BEFORE INSERT OR UPDATE ON key
+    FOR EACH ROW
+    EXECUTE FUNCTION validate_key_owner();
+
+-- Trigger for updated timestamp
+CREATE TRIGGER update_key_updated
+    BEFORE UPDATE ON key
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE key IS 'Keys store configuration values and secrets with ownership scoping';
+COMMENT ON COLUMN key.ref IS 'Unique key reference (format: [owner.]name)';
+COMMENT ON COLUMN key.owner_type IS 'Type of owner (system, identity, pack, action, sensor)';
+COMMENT ON COLUMN key.owner IS 'Owner identifier (auto-populated by trigger)';
+COMMENT ON COLUMN key.owner_identity IS 'Identity owner (if owner_type=identity)';
+COMMENT ON COLUMN key.owner_pack IS 'Pack owner (if owner_type=pack)';
+COMMENT ON COLUMN key.owner_pack_ref IS 'Pack reference for owner_pack';
+COMMENT ON COLUMN key.owner_action IS 'Action owner (if owner_type=action)';
+COMMENT ON COLUMN key.owner_sensor IS 'Sensor owner (if owner_type=sensor)';
+COMMENT ON COLUMN key.name IS 'Key name within owner scope';
+COMMENT ON COLUMN key.encrypted IS 'Whether the value is encrypted';
+COMMENT ON COLUMN key.encryption_key_hash IS 'Hash of encryption key used';
+COMMENT ON COLUMN key.value IS 'The actual value (encrypted if encrypted=true)';
+
+
+-- Add foreign key constraints for action and sensor references
+ALTER TABLE key
+    ADD CONSTRAINT key_owner_action_fkey
+    FOREIGN KEY (owner_action) REFERENCES action(id) ON DELETE CASCADE;
+
+ALTER TABLE key
+    ADD CONSTRAINT key_owner_sensor_fkey
+    FOREIGN KEY (owner_sensor) REFERENCES sensor(id) ON DELETE CASCADE;
+
+-- ============================================================================
+-- ARTIFACT TABLE
+-- ============================================================================
+
+CREATE TABLE artifact (
+    id BIGSERIAL PRIMARY KEY,
+    ref TEXT NOT NULL,
+    scope owner_type_enum NOT NULL DEFAULT 'system',
+    owner TEXT NOT NULL DEFAULT '',
+    type artifact_type_enum NOT NULL,
+    visibility artifact_visibility_enum NOT NULL DEFAULT 'private',
+    retention_policy artifact_retention_enum NOT NULL DEFAULT 'versions',
+    retention_limit INTEGER NOT NULL DEFAULT 1,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_artifact_ref ON artifact(ref);
+CREATE INDEX idx_artifact_scope ON artifact(scope);
+CREATE INDEX idx_artifact_owner ON artifact(owner);
+CREATE INDEX idx_artifact_type ON artifact(type);
+CREATE INDEX idx_artifact_created ON artifact(created DESC);
+CREATE INDEX idx_artifact_scope_owner ON artifact(scope, owner);
+CREATE INDEX idx_artifact_type_created ON artifact(type, created DESC);
+CREATE INDEX idx_artifact_visibility ON artifact(visibility);
+CREATE INDEX idx_artifact_visibility_scope ON artifact(visibility, scope, owner);
+
+-- Trigger
+CREATE TRIGGER update_artifact_updated
+    BEFORE UPDATE ON artifact
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE artifact IS 'Artifacts track files, logs, and outputs from executions';
+COMMENT ON COLUMN artifact.ref IS 'Artifact reference/path';
+COMMENT ON COLUMN artifact.scope IS 'Owner type (system, identity, pack, action, sensor)';
+COMMENT ON COLUMN artifact.owner IS 'Owner identifier';
+COMMENT ON COLUMN artifact.type IS 'Artifact type (file, url, progress, etc.)';
+COMMENT ON COLUMN artifact.visibility IS 'Visibility level: public (all users) or private (scoped by scope/owner)';
+COMMENT ON COLUMN artifact.retention_policy IS 'How to retain artifacts (versions, days, hours, minutes)';
+COMMENT ON COLUMN artifact.retention_limit IS 'Numeric limit for retention policy';
+
+-- ============================================================================
+-- QUEUE_STATS TABLE
+-- ============================================================================
+
+CREATE TABLE queue_stats (
+    action_id BIGINT PRIMARY KEY REFERENCES action(id) ON DELETE CASCADE,
+    queue_length INTEGER NOT NULL DEFAULT 0,
+    active_count INTEGER NOT NULL DEFAULT 0,
+    max_concurrent INTEGER NOT NULL DEFAULT 1,
+    oldest_enqueued_at TIMESTAMPTZ,
+    total_enqueued BIGINT NOT NULL DEFAULT 0,
+    total_completed BIGINT NOT NULL DEFAULT 0,
+    last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Indexes
+CREATE INDEX idx_queue_stats_last_updated ON queue_stats(last_updated);
+
+-- Comments
+COMMENT ON TABLE queue_stats IS 'Real-time queue statistics for action execution ordering';
+COMMENT ON COLUMN queue_stats.action_id IS 'Foreign key to action table';
+COMMENT ON COLUMN queue_stats.queue_length IS 'Number of executions waiting in queue';
+COMMENT ON COLUMN queue_stats.active_count IS 'Number of currently running executions';
+COMMENT ON COLUMN queue_stats.max_concurrent IS 'Maximum concurrent executions allowed';
+COMMENT ON COLUMN queue_stats.oldest_enqueued_at IS 'Timestamp of oldest queued execution (NULL if queue empty)';
+COMMENT ON COLUMN queue_stats.total_enqueued IS 'Total executions enqueued since queue creation';
+COMMENT ON COLUMN queue_stats.total_completed IS 'Total executions completed since queue creation';
+COMMENT ON COLUMN queue_stats.last_updated IS 'Timestamp of last statistics update';
+
+-- ============================================================================
+-- PACK ENVIRONMENT TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS pack_environment (
+    id BIGSERIAL PRIMARY KEY,
+    pack BIGINT NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+    pack_ref TEXT NOT NULL,
+    runtime BIGINT NOT NULL REFERENCES runtime(id) ON DELETE CASCADE,
+    runtime_ref TEXT NOT NULL,
+    env_path TEXT NOT NULL,
+    status pack_environment_status_enum NOT NULL DEFAULT 'pending',
+    installed_at TIMESTAMPTZ,
+    last_verified TIMESTAMPTZ,
+    install_log TEXT,
+    install_error TEXT,
+    metadata JSONB DEFAULT '{}'::jsonb,
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    UNIQUE(pack, runtime)
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_pack_environment_pack ON pack_environment(pack);
+CREATE INDEX IF NOT EXISTS idx_pack_environment_runtime ON pack_environment(runtime);
+CREATE INDEX IF NOT EXISTS idx_pack_environment_status ON pack_environment(status);
+CREATE INDEX IF NOT EXISTS idx_pack_environment_pack_ref ON pack_environment(pack_ref);
+CREATE INDEX IF NOT EXISTS idx_pack_environment_runtime_ref ON pack_environment(runtime_ref);
+CREATE INDEX IF NOT EXISTS idx_pack_environment_pack_runtime ON pack_environment(pack, runtime);
+
+-- Trigger for updated timestamp
+CREATE TRIGGER update_pack_environment_updated
+    BEFORE UPDATE ON pack_environment
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_column();
+
+-- Comments
+COMMENT ON TABLE pack_environment IS 'Tracks pack-specific runtime environments for dependency isolation';
+COMMENT ON COLUMN pack_environment.pack IS 'Pack that owns this environment';
+COMMENT ON COLUMN pack_environment.pack_ref IS 'Pack reference for quick lookup';
+COMMENT ON COLUMN pack_environment.runtime IS 'Runtime used for this environment';
+COMMENT ON COLUMN pack_environment.runtime_ref IS 'Runtime reference for quick lookup';
+COMMENT ON COLUMN pack_environment.env_path IS 'Filesystem path to the environment directory (e.g., /opt/attune/packenvs/mypack/python)';
+COMMENT ON COLUMN pack_environment.status IS 'Current installation status';
+COMMENT ON COLUMN pack_environment.installed_at IS 'When the environment was successfully installed';
+COMMENT ON COLUMN pack_environment.last_verified IS 'Last time the environment was verified as working';
+COMMENT ON COLUMN pack_environment.install_log IS 'Installation output logs';
+COMMENT ON COLUMN pack_environment.install_error IS 'Error message if installation failed';
+COMMENT ON COLUMN pack_environment.metadata IS 'Additional metadata (installed packages, versions, etc.)';
+
+-- ============================================================================
+-- PACK ENVIRONMENT: Update existing runtimes with installer metadata
+-- ============================================================================
+
+-- Python runtime installers
+UPDATE runtime
+SET installers = jsonb_build_object(
+    'base_path_template', '/opt/attune/packenvs/{pack_ref}/{runtime_name_lower}',
+    'installers', jsonb_build_array(
+        jsonb_build_object(
+            'name', 'create_venv',
+            'description', 'Create Python virtual environment',
+            'command', 'python3',
+            'args', jsonb_build_array('-m', 'venv', '{env_path}'),
+            'cwd', '{pack_path}',
+            'env', jsonb_build_object(),
+            'order', 1,
+            'optional', false
+        ),
+        jsonb_build_object(
+            'name', 'upgrade_pip',
+            'description', 'Upgrade pip to latest version',
+            'command', '{env_path}/bin/pip',
+            'args', jsonb_build_array('install', '--upgrade', 'pip'),
+            'cwd', '{pack_path}',
+            'env', jsonb_build_object(),
+            'order', 2,
+            'optional', true
+        ),
+        jsonb_build_object(
+            'name', 'install_requirements',
+            'description', 'Install pack Python dependencies',
+            'command', '{env_path}/bin/pip',
+            'args', jsonb_build_array('install', '-r', '{pack_path}/requirements.txt'),
+            'cwd', '{pack_path}',
+            'env', jsonb_build_object(),
+            'order', 3,
+            'optional', false,
+            'condition', jsonb_build_object(
+                'file_exists', '{pack_path}/requirements.txt'
+            )
+        )
+    ),
+    'executable_templates', jsonb_build_object(
+        'python', '{env_path}/bin/python',
+        'pip', '{env_path}/bin/pip'
+    )
+)
+WHERE ref = 'core.python';
+
+-- Node.js runtime installers
+UPDATE runtime
+SET installers = jsonb_build_object(
+    'base_path_template', '/opt/attune/packenvs/{pack_ref}/{runtime_name_lower}',
+    'installers', jsonb_build_array(
+        jsonb_build_object(
+            'name', 'npm_install',
+            'description', 'Install Node.js dependencies',
+            'command', 'npm',
+            'args', jsonb_build_array('install', '--prefix', '{env_path}'),
+            'cwd', '{pack_path}',
+            'env', jsonb_build_object(
+                'NODE_PATH', '{env_path}/node_modules'
+            ),
+            'order', 1,
+            'optional', false,
+            'condition', jsonb_build_object(
+                'file_exists', '{pack_path}/package.json'
+            )
+        )
+    ),
+    'executable_templates', jsonb_build_object(
+        'node', 'node',
+        'npm', 'npm'
+    ),
+    'env_vars', jsonb_build_object(
+        'NODE_PATH', '{env_path}/node_modules'
+    )
+)
+WHERE ref = 'core.nodejs';
+
+-- Shell runtime (no environment needed, uses system shell)
+UPDATE runtime
+SET installers = jsonb_build_object(
+    'base_path_template', '/opt/attune/packenvs/{pack_ref}/{runtime_name_lower}',
+    'installers', jsonb_build_array(),
+    'executable_templates', jsonb_build_object(
+        'sh', 'sh',
+        'bash', 'bash'
+    ),
+    'requires_environment', false
+)
+WHERE ref = 'core.shell';
+
+-- Native runtime (no environment needed, binaries are standalone)
+UPDATE runtime
+SET installers = jsonb_build_object(
+    'base_path_template', '/opt/attune/packenvs/{pack_ref}/{runtime_name_lower}',
+    'installers', jsonb_build_array(),
+    'executable_templates', jsonb_build_object(),
+    'requires_environment', false
+)
+WHERE ref = 'core.native';
+
+-- Built-in sensor runtime (internal, no environment)
+UPDATE runtime
+SET installers = jsonb_build_object(
+    'installers', jsonb_build_array(),
+    'requires_environment', false
+)
+WHERE ref = 'core.sensor.builtin';
+
+-- ============================================================================
+-- PACK ENVIRONMENT: Helper functions
+-- ============================================================================
+
+-- Function to get environment path for a pack/runtime combination
+CREATE OR REPLACE FUNCTION get_pack_environment_path(p_pack_ref TEXT, p_runtime_ref TEXT)
+RETURNS TEXT AS $$
+DECLARE
+    v_runtime_name TEXT;
+    v_base_template TEXT;
+    v_result TEXT;
+BEGIN
+    -- Get runtime name and base path template
+    SELECT
+        LOWER(name),
+        installers->>'base_path_template'
+    INTO v_runtime_name, v_base_template
+    FROM runtime
+    WHERE ref = p_runtime_ref;
+
+    IF v_base_template IS NULL THEN
+        v_base_template := '/opt/attune/packenvs/{pack_ref}/{runtime_name_lower}';
+    END IF;
+
+    -- Replace template variables
+    v_result := v_base_template;
+    v_result := REPLACE(v_result, '{pack_ref}', p_pack_ref);
+    v_result := REPLACE(v_result, '{runtime_ref}', p_runtime_ref);
+    v_result := REPLACE(v_result, '{runtime_name_lower}', v_runtime_name);
+
+    RETURN v_result;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION get_pack_environment_path IS 'Calculate the filesystem path for a pack runtime environment';
+
+-- Function to check if a runtime requires an environment
+CREATE OR REPLACE FUNCTION runtime_requires_environment(p_runtime_ref TEXT)
+RETURNS BOOLEAN AS $$
+DECLARE
+    v_requires BOOLEAN;
+BEGIN
+    SELECT COALESCE((installers->>'requires_environment')::boolean, true)
+    INTO v_requires
+    FROM runtime
+    WHERE ref = p_runtime_ref;
+
+    RETURN COALESCE(v_requires, false);
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION runtime_requires_environment IS 'Check if a runtime needs a pack-specific environment';
+
+-- ============================================================================
+-- PACK ENVIRONMENT: Status view
+-- ============================================================================
+
+CREATE OR REPLACE VIEW v_pack_environment_status AS
+SELECT
+    pe.id,
+    pe.pack,
+    p.ref AS pack_ref,
+    p.label AS pack_name,
+    pe.runtime,
+    r.ref AS runtime_ref,
+    r.name AS runtime_name,
+    pe.env_path,
+    pe.status,
+    pe.installed_at,
+    pe.last_verified,
+    CASE
+        WHEN pe.status = 'ready' AND pe.last_verified < NOW() - INTERVAL '7 days' THEN true
+        ELSE false
+    END AS needs_verification,
+    CASE
+        WHEN pe.status = 'ready' THEN 'healthy'
+        WHEN pe.status = 'failed' THEN 'unhealthy'
+        WHEN pe.status IN ('pending', 'installing') THEN 'provisioning'
+        WHEN pe.status = 'outdated' THEN 'needs_update'
+        ELSE 'unknown'
+    END AS health_status,
+    pe.install_error,
+    pe.created,
+    pe.updated
+FROM pack_environment pe
+JOIN pack p ON pe.pack = p.id
+JOIN runtime r ON pe.runtime = r.id;
+
+COMMENT ON VIEW v_pack_environment_status IS 'Consolidated view of pack environment status with health indicators';
+
+-- ============================================================================
+-- PACK TEST EXECUTION TABLE
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS pack_test_execution (
+    id BIGSERIAL PRIMARY KEY,
+    pack_id BIGINT NOT NULL REFERENCES pack(id) ON DELETE CASCADE,
+    pack_version VARCHAR(50) NOT NULL,
+    execution_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    trigger_reason VARCHAR(50) NOT NULL, -- 'install', 'update', 'manual', 'validation'
+    total_tests INT NOT NULL,
+    passed INT NOT NULL,
+    failed INT NOT NULL,
+    skipped INT NOT NULL,
+    pass_rate DECIMAL(5,4) NOT NULL, -- 0.0000 to 1.0000
+    duration_ms BIGINT NOT NULL,
+    result JSONB NOT NULL, -- Full test result structure
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT valid_test_counts CHECK (total_tests >= 0 AND passed >= 0 AND failed >= 0 AND skipped >= 0),
+    CONSTRAINT valid_pass_rate CHECK (pass_rate >= 0.0 AND pass_rate <= 1.0),
+    CONSTRAINT valid_trigger_reason CHECK (trigger_reason IN ('install', 'update', 'manual', 'validation'))
+);
+
+-- Indexes for efficient queries
+CREATE INDEX idx_pack_test_execution_pack_id ON pack_test_execution(pack_id);
+CREATE INDEX idx_pack_test_execution_time ON pack_test_execution(execution_time DESC);
+CREATE INDEX idx_pack_test_execution_pass_rate ON pack_test_execution(pass_rate);
+CREATE INDEX idx_pack_test_execution_trigger ON pack_test_execution(trigger_reason);
+
+-- Comments for documentation
+COMMENT ON TABLE pack_test_execution IS 'Tracks pack test execution results for validation and auditing';
+COMMENT ON COLUMN pack_test_execution.pack_id IS 'Reference to the pack being tested';
+COMMENT ON COLUMN pack_test_execution.pack_version IS 'Version of the pack at test time';
+COMMENT ON COLUMN pack_test_execution.trigger_reason IS 'What triggered the test: install, update, manual, validation';
+COMMENT ON COLUMN pack_test_execution.pass_rate IS 'Percentage of tests passed (0.0 to 1.0)';
+COMMENT ON COLUMN pack_test_execution.result IS 'Full JSON structure with detailed test results';
+
+-- Pack test result summary view (all test executions with pack info)
+CREATE OR REPLACE VIEW pack_test_summary AS
+SELECT
+    p.id AS pack_id,
+    p.ref AS pack_ref,
+    p.label AS pack_label,
+    pte.id AS test_execution_id,
+    pte.pack_version,
+    pte.execution_time AS test_time,
+    pte.trigger_reason,
+    pte.total_tests,
+    pte.passed,
+    pte.failed,
+    pte.skipped,
+    pte.pass_rate,
+    pte.duration_ms,
+    ROW_NUMBER() OVER (PARTITION BY p.id ORDER BY pte.execution_time DESC) AS rn
+FROM pack p
+LEFT JOIN pack_test_execution pte ON p.id = pte.pack_id
+WHERE pte.id IS NOT NULL;
+
+COMMENT ON VIEW pack_test_summary IS 'Summary of all pack test executions with pack details';
+
+-- Latest test results per pack view
+CREATE OR REPLACE VIEW pack_latest_test AS
+SELECT
+    pack_id,
+    pack_ref,
+    pack_label,
+    test_execution_id,
+    pack_version,
+    test_time,
+    trigger_reason,
+    total_tests,
+    passed,
+    failed,
+    skipped,
+    pass_rate,
+    duration_ms
+FROM pack_test_summary
+WHERE rn = 1;
+
+COMMENT ON VIEW pack_latest_test IS 'Latest test results for each pack';
+
+-- Function to get pack test statistics
+CREATE OR REPLACE FUNCTION get_pack_test_stats(p_pack_id BIGINT)
+RETURNS TABLE (
+    total_executions BIGINT,
+    successful_executions BIGINT,
+    failed_executions BIGINT,
+    avg_pass_rate DECIMAL,
+    avg_duration_ms BIGINT,
+    last_test_time TIMESTAMPTZ,
+    last_test_passed BOOLEAN
+) AS $$
+BEGIN
+    RETURN QUERY
+    SELECT
+        COUNT(*)::BIGINT AS total_executions,
+        COUNT(*) FILTER (WHERE passed = total_tests)::BIGINT AS successful_executions,
+        COUNT(*) FILTER (WHERE failed > 0)::BIGINT AS failed_executions,
+        AVG(pass_rate) AS avg_pass_rate,
+        AVG(duration_ms)::BIGINT AS avg_duration_ms,
+        MAX(execution_time) AS last_test_time,
+        (SELECT failed = 0 FROM pack_test_execution
+         WHERE pack_id = p_pack_id
+         ORDER BY execution_time DESC
+         LIMIT 1) AS last_test_passed
+    FROM pack_test_execution
+    WHERE pack_id = p_pack_id;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION get_pack_test_stats IS 'Get statistical summary of test executions for a pack';
+
+-- Function to check if pack has recent passing tests
+CREATE OR REPLACE FUNCTION pack_has_passing_tests(
+    p_pack_id BIGINT,
+    p_hours_ago INT DEFAULT 24
+)
+RETURNS BOOLEAN AS $$
+DECLARE
+    v_has_passing_tests BOOLEAN;
+BEGIN
+    SELECT EXISTS(
+        SELECT 1
+        FROM pack_test_execution
+        WHERE pack_id = p_pack_id
+        AND execution_time > NOW() - (p_hours_ago || ' hours')::INTERVAL
+        AND failed = 0
+        AND total_tests > 0
+    ) INTO v_has_passing_tests;
+
+    RETURN v_has_passing_tests;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION pack_has_passing_tests IS 'Check if pack has recent passing test executions';
+
+-- Add trigger to update pack metadata on test execution
+CREATE OR REPLACE FUNCTION update_pack_test_metadata()
+RETURNS TRIGGER AS $$
+BEGIN
+    -- Could update pack table with last_tested timestamp if we add that column
+    -- For now, just a placeholder for future functionality
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_update_pack_test_metadata
+    AFTER INSERT ON pack_test_execution
+    FOR EACH ROW
+    EXECUTE FUNCTION update_pack_test_metadata();
+
+COMMENT ON TRIGGER trigger_update_pack_test_metadata ON pack_test_execution IS 'Updates pack metadata when tests are executed';
+
+-- ============================================================================
+-- WEBHOOK FUNCTIONS
+-- ============================================================================
+
+-- Drop existing functions to avoid signature conflicts
+DROP FUNCTION IF EXISTS enable_trigger_webhook(BIGINT, JSONB);
+DROP FUNCTION IF EXISTS enable_trigger_webhook(BIGINT);
+DROP FUNCTION IF EXISTS disable_trigger_webhook(BIGINT);
+DROP FUNCTION IF EXISTS regenerate_trigger_webhook_key(BIGINT);
+
+-- Function to enable webhooks for a trigger
+CREATE OR REPLACE FUNCTION enable_trigger_webhook(
+    p_trigger_id BIGINT,
+    p_config JSONB DEFAULT '{}'::jsonb
+)
+RETURNS TABLE(
+    webhook_enabled BOOLEAN,
+    webhook_key VARCHAR(255),
+    webhook_url TEXT
+) AS $$
+DECLARE
+    v_webhook_key VARCHAR(255);
+    v_api_base_url TEXT := 'http://localhost:8080'; -- Default, should be configured
+BEGIN
+    -- Check if trigger exists
+    IF NOT EXISTS (SELECT 1 FROM trigger WHERE id = p_trigger_id) THEN
+        RAISE EXCEPTION 'Trigger with id % does not exist', p_trigger_id;
+    END IF;
+
+    -- Generate webhook key if one doesn't exist
+    SELECT t.webhook_key INTO v_webhook_key
+    FROM trigger t
+    WHERE t.id = p_trigger_id;
+
+    IF v_webhook_key IS NULL THEN
+        v_webhook_key := generate_webhook_key();
+    END IF;
+
+    -- Update trigger to enable webhooks
+    UPDATE trigger
+    SET
+        webhook_enabled = TRUE,
+        webhook_key = v_webhook_key,
+        webhook_config = p_config,
+        updated = NOW()
+    WHERE id = p_trigger_id;
+
+    -- Return webhook details
+    RETURN QUERY SELECT
+        TRUE,
+        v_webhook_key,
+        v_api_base_url || '/api/v1/webhooks/' || v_webhook_key;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION enable_trigger_webhook(BIGINT, JSONB) IS
+    'Enables webhooks for a trigger with optional configuration. Generates a new webhook key if one does not exist. Returns webhook details.';
+
+-- Function to disable webhooks for a trigger
+CREATE OR REPLACE FUNCTION disable_trigger_webhook(
+    p_trigger_id BIGINT
+)
+RETURNS BOOLEAN AS $$
+BEGIN
+    -- Check if trigger exists
+    IF NOT EXISTS (SELECT 1 FROM trigger WHERE id = p_trigger_id) THEN
+        RAISE EXCEPTION 'Trigger with id % does not exist', p_trigger_id;
+    END IF;
+
+    -- Update trigger to disable webhooks
+    -- Set webhook_key to NULL when disabling to remove it from API responses
+    UPDATE trigger
+    SET
+        webhook_enabled = FALSE,
+        webhook_key = NULL,
+        updated = NOW()
+    WHERE id = p_trigger_id;
+
+    RETURN TRUE;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION disable_trigger_webhook(BIGINT) IS
+    'Disables webhooks for a trigger. Webhook key is removed when disabled.';
+
+-- Function to regenerate webhook key for a trigger
+CREATE OR REPLACE FUNCTION regenerate_trigger_webhook_key(
+    p_trigger_id BIGINT
+)
+RETURNS TABLE(
+    webhook_key VARCHAR(255),
+    previous_key_revoked BOOLEAN
+) AS $$
+DECLARE
+    v_new_key VARCHAR(255);
+    v_old_key VARCHAR(255);
+    v_webhook_enabled BOOLEAN;
+BEGIN
+    -- Check if trigger exists
+    IF NOT EXISTS (SELECT 1 FROM trigger WHERE id = p_trigger_id) THEN
+        RAISE EXCEPTION 'Trigger with id % does not exist', p_trigger_id;
+    END IF;
+
+    -- Get current webhook state
+    SELECT t.webhook_key, t.webhook_enabled INTO v_old_key, v_webhook_enabled
+    FROM trigger t
+    WHERE t.id = p_trigger_id;
+
+    -- Check if webhooks are enabled
+    IF NOT v_webhook_enabled THEN
+        RAISE EXCEPTION 'Webhooks are not enabled for trigger %', p_trigger_id;
+    END IF;
+
+    -- Generate new key
+    v_new_key := generate_webhook_key();
+
+    -- Update trigger with new key
+    UPDATE trigger
+    SET
+        webhook_key = v_new_key,
+        updated = NOW()
+    WHERE id = p_trigger_id;
+
+    -- Return new key and whether old key was present
+    RETURN QUERY SELECT
+        v_new_key,
+        (v_old_key IS NOT NULL);
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION regenerate_trigger_webhook_key(BIGINT) IS
+    'Regenerates webhook key for a trigger. Returns new key and whether a previous key was revoked.';
+
+-- Verify all webhook functions exist
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_proc p
+        JOIN pg_namespace n ON p.pronamespace = n.oid
+        WHERE n.nspname = current_schema()
+        AND p.proname = 'enable_trigger_webhook'
+    ) THEN
+        RAISE EXCEPTION 'enable_trigger_webhook function not found after migration';
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_proc p
+        JOIN pg_namespace n ON p.pronamespace = n.oid
+        WHERE n.nspname = current_schema()
+        AND p.proname = 'disable_trigger_webhook'
+    ) THEN
+        RAISE EXCEPTION 'disable_trigger_webhook function not found after migration';
+    END IF;
+
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_proc p
+        JOIN pg_namespace n ON p.pronamespace = n.oid
+        WHERE n.nspname = current_schema()
+        AND p.proname = 'regenerate_trigger_webhook_key'
+    ) THEN
+        RAISE EXCEPTION 'regenerate_trigger_webhook_key function not found after migration';
+    END IF;
+
+    RAISE NOTICE 'All webhook functions successfully created';
+END $$;

docker/distributable/migrations/20250101000008_notify_triggers.sql

@@ -0,0 +1,428 @@
+-- Migration: LISTEN/NOTIFY Triggers
+-- Description: Consolidated PostgreSQL LISTEN/NOTIFY triggers for real-time event notifications
+-- Version: 20250101000008
+
+-- ============================================================================
+-- EXECUTION CHANGE NOTIFICATION
+-- ============================================================================
+
+-- Function to notify on execution creation
+CREATE OR REPLACE FUNCTION notify_execution_created()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+    enforcement_rule_ref TEXT;
+    enforcement_trigger_ref TEXT;
+BEGIN
+    -- Lookup enforcement details if this execution is linked to an enforcement
+    IF NEW.enforcement IS NOT NULL THEN
+        SELECT rule_ref, trigger_ref
+        INTO enforcement_rule_ref, enforcement_trigger_ref
+        FROM enforcement
+        WHERE id = NEW.enforcement;
+    END IF;
+
+    payload := json_build_object(
+        'entity_type', 'execution',
+        'entity_id', NEW.id,
+        'id', NEW.id,
+        'action_id', NEW.action,
+        'action_ref', NEW.action_ref,
+        'status', NEW.status,
+        'enforcement', NEW.enforcement,
+        'rule_ref', enforcement_rule_ref,
+        'trigger_ref', enforcement_trigger_ref,
+        'parent', NEW.parent,
+        'result', NEW.result,
+        'started_at', NEW.started_at,
+        'workflow_task', NEW.workflow_task,
+        'created', NEW.created,
+        'updated', NEW.updated
+    );
+
+    PERFORM pg_notify('execution_created', payload::text);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to notify on execution status changes
+CREATE OR REPLACE FUNCTION notify_execution_status_changed()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+    enforcement_rule_ref TEXT;
+    enforcement_trigger_ref TEXT;
+BEGIN
+    -- Only notify on updates, not inserts
+    IF TG_OP = 'UPDATE' AND OLD.status IS DISTINCT FROM NEW.status THEN
+        -- Lookup enforcement details if this execution is linked to an enforcement
+        IF NEW.enforcement IS NOT NULL THEN
+            SELECT rule_ref, trigger_ref
+            INTO enforcement_rule_ref, enforcement_trigger_ref
+            FROM enforcement
+            WHERE id = NEW.enforcement;
+        END IF;
+
+        payload := json_build_object(
+            'entity_type', 'execution',
+            'entity_id', NEW.id,
+            'id', NEW.id,
+            'action_id', NEW.action,
+            'action_ref', NEW.action_ref,
+            'status', NEW.status,
+            'old_status', OLD.status,
+            'enforcement', NEW.enforcement,
+            'rule_ref', enforcement_rule_ref,
+            'trigger_ref', enforcement_trigger_ref,
+            'parent', NEW.parent,
+            'result', NEW.result,
+            'started_at', NEW.started_at,
+            'workflow_task', NEW.workflow_task,
+            'created', NEW.created,
+            'updated', NEW.updated
+        );
+
+        PERFORM pg_notify('execution_status_changed', payload::text);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on execution table for creation
+CREATE TRIGGER execution_created_notify
+    AFTER INSERT ON execution
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_execution_created();
+
+-- Trigger on execution table for status changes
+CREATE TRIGGER execution_status_changed_notify
+    AFTER UPDATE ON execution
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_execution_status_changed();
+
+COMMENT ON FUNCTION notify_execution_created() IS 'Sends execution creation notifications via PostgreSQL LISTEN/NOTIFY';
+COMMENT ON FUNCTION notify_execution_status_changed() IS 'Sends execution status change notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- ============================================================================
+-- EVENT CREATION NOTIFICATION
+-- ============================================================================
+
+-- Function to notify on event creation
+CREATE OR REPLACE FUNCTION notify_event_created()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    payload := json_build_object(
+        'entity_type', 'event',
+        'entity_id', NEW.id,
+        'id', NEW.id,
+        'trigger', NEW.trigger,
+        'trigger_ref', NEW.trigger_ref,
+        'source', NEW.source,
+        'source_ref', NEW.source_ref,
+        'rule', NEW.rule,
+        'rule_ref', NEW.rule_ref,
+        'payload', NEW.payload,
+        'created', NEW.created
+    );
+
+    PERFORM pg_notify('event_created', payload::text);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on event table
+CREATE TRIGGER event_created_notify
+    AFTER INSERT ON event
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_event_created();
+
+COMMENT ON FUNCTION notify_event_created() IS 'Sends event creation notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- ============================================================================
+-- ENFORCEMENT CHANGE NOTIFICATION
+-- ============================================================================
+
+-- Function to notify on enforcement creation
+CREATE OR REPLACE FUNCTION notify_enforcement_created()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    payload := json_build_object(
+        'entity_type', 'enforcement',
+        'entity_id', NEW.id,
+        'id', NEW.id,
+        'rule', NEW.rule,
+        'rule_ref', NEW.rule_ref,
+        'trigger_ref', NEW.trigger_ref,
+        'event', NEW.event,
+        'status', NEW.status,
+        'condition', NEW.condition,
+        'conditions', NEW.conditions,
+        'config', NEW.config,
+        'payload', NEW.payload,
+        'created', NEW.created,
+        'resolved_at', NEW.resolved_at
+    );
+
+    PERFORM pg_notify('enforcement_created', payload::text);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on enforcement table
+CREATE TRIGGER enforcement_created_notify
+    AFTER INSERT ON enforcement
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_enforcement_created();
+
+COMMENT ON FUNCTION notify_enforcement_created() IS 'Sends enforcement creation notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- Function to notify on enforcement status changes
+CREATE OR REPLACE FUNCTION notify_enforcement_status_changed()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    -- Only notify on updates when status actually changed
+    IF TG_OP = 'UPDATE' AND OLD.status IS DISTINCT FROM NEW.status THEN
+        payload := json_build_object(
+            'entity_type', 'enforcement',
+            'entity_id', NEW.id,
+            'id', NEW.id,
+            'rule', NEW.rule,
+            'rule_ref', NEW.rule_ref,
+            'trigger_ref', NEW.trigger_ref,
+            'event', NEW.event,
+            'status', NEW.status,
+            'old_status', OLD.status,
+            'condition', NEW.condition,
+            'conditions', NEW.conditions,
+            'config', NEW.config,
+            'payload', NEW.payload,
+            'created', NEW.created,
+            'resolved_at', NEW.resolved_at
+        );
+
+        PERFORM pg_notify('enforcement_status_changed', payload::text);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on enforcement table for status changes
+CREATE TRIGGER enforcement_status_changed_notify
+    AFTER UPDATE ON enforcement
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_enforcement_status_changed();
+
+COMMENT ON FUNCTION notify_enforcement_status_changed() IS 'Sends enforcement status change notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- ============================================================================
+-- INQUIRY NOTIFICATIONS
+-- ============================================================================
+
+-- Function to notify on inquiry creation
+CREATE OR REPLACE FUNCTION notify_inquiry_created()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    payload := json_build_object(
+        'entity_type', 'inquiry',
+        'entity_id', NEW.id,
+        'id', NEW.id,
+        'execution', NEW.execution,
+        'status', NEW.status,
+        'ttl', NEW.ttl,
+        'created', NEW.created
+    );
+
+    PERFORM pg_notify('inquiry_created', payload::text);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Function to notify on inquiry response
+CREATE OR REPLACE FUNCTION notify_inquiry_responded()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    -- Only notify when status changes to 'responded'
+    IF TG_OP = 'UPDATE' AND NEW.status = 'responded' AND OLD.status != 'responded' THEN
+        payload := json_build_object(
+            'entity_type', 'inquiry',
+            'entity_id', NEW.id,
+            'id', NEW.id,
+            'execution', NEW.execution,
+            'status', NEW.status,
+            'response', NEW.response,
+            'updated', NEW.updated
+        );
+
+        PERFORM pg_notify('inquiry_responded', payload::text);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on inquiry table for creation
+CREATE TRIGGER inquiry_created_notify
+    AFTER INSERT ON inquiry
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_inquiry_created();
+
+-- Trigger on inquiry table for responses
+CREATE TRIGGER inquiry_responded_notify
+    AFTER UPDATE ON inquiry
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_inquiry_responded();
+
+COMMENT ON FUNCTION notify_inquiry_created() IS 'Sends inquiry creation notifications via PostgreSQL LISTEN/NOTIFY';
+COMMENT ON FUNCTION notify_inquiry_responded() IS 'Sends inquiry response notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- ============================================================================
+-- WORKFLOW EXECUTION NOTIFICATIONS
+-- ============================================================================
+
+-- Function to notify on workflow execution status changes
+CREATE OR REPLACE FUNCTION notify_workflow_execution_status_changed()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    -- Only notify for workflow executions when status changes
+    IF TG_OP = 'UPDATE' AND NEW.is_workflow = true AND OLD.status IS DISTINCT FROM NEW.status THEN
+        payload := json_build_object(
+            'entity_type', 'execution',
+            'entity_id', NEW.id,
+            'id', NEW.id,
+            'action_ref', NEW.action_ref,
+            'status', NEW.status,
+            'old_status', OLD.status,
+            'workflow_def', NEW.workflow_def,
+            'parent', NEW.parent,
+            'created', NEW.created,
+            'updated', NEW.updated
+        );
+
+        PERFORM pg_notify('workflow_execution_status_changed', payload::text);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on execution table for workflow status changes
+CREATE TRIGGER workflow_execution_status_changed_notify
+    AFTER UPDATE ON execution
+    FOR EACH ROW
+    WHEN (NEW.is_workflow = true)
+    EXECUTE FUNCTION notify_workflow_execution_status_changed();
+
+COMMENT ON FUNCTION notify_workflow_execution_status_changed() IS 'Sends workflow execution status change notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- ============================================================================
+-- ARTIFACT NOTIFICATIONS
+-- ============================================================================
+
+-- Function to notify on artifact creation
+CREATE OR REPLACE FUNCTION notify_artifact_created()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+BEGIN
+    payload := json_build_object(
+        'entity_type', 'artifact',
+        'entity_id', NEW.id,
+        'id', NEW.id,
+        'ref', NEW.ref,
+        'type', NEW.type,
+        'visibility', NEW.visibility,
+        'name', NEW.name,
+        'execution', NEW.execution,
+        'scope', NEW.scope,
+        'owner', NEW.owner,
+        'content_type', NEW.content_type,
+        'size_bytes', NEW.size_bytes,
+        'created', NEW.created
+    );
+
+    PERFORM pg_notify('artifact_created', payload::text);
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on artifact table for creation
+CREATE TRIGGER artifact_created_notify
+    AFTER INSERT ON artifact
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_artifact_created();
+
+COMMENT ON FUNCTION notify_artifact_created() IS 'Sends artifact creation notifications via PostgreSQL LISTEN/NOTIFY';
+
+-- Function to notify on artifact updates (progress appends, data changes)
+CREATE OR REPLACE FUNCTION notify_artifact_updated()
+RETURNS TRIGGER AS $$
+DECLARE
+    payload JSON;
+    latest_percent DOUBLE PRECISION;
+    latest_message TEXT;
+    entry_count INTEGER;
+BEGIN
+    -- Only notify on actual changes
+    IF TG_OP = 'UPDATE' THEN
+        -- Extract progress summary from data array if this is a progress artifact
+        IF NEW.type = 'progress' AND NEW.data IS NOT NULL AND jsonb_typeof(NEW.data) = 'array' THEN
+            entry_count := jsonb_array_length(NEW.data);
+            IF entry_count > 0 THEN
+                latest_percent := (NEW.data -> (entry_count - 1) ->> 'percent')::DOUBLE PRECISION;
+                latest_message := NEW.data -> (entry_count - 1) ->> 'message';
+            END IF;
+        END IF;
+
+        payload := json_build_object(
+            'entity_type', 'artifact',
+            'entity_id', NEW.id,
+            'id', NEW.id,
+            'ref', NEW.ref,
+            'type', NEW.type,
+            'visibility', NEW.visibility,
+            'name', NEW.name,
+            'execution', NEW.execution,
+            'scope', NEW.scope,
+            'owner', NEW.owner,
+            'content_type', NEW.content_type,
+            'size_bytes', NEW.size_bytes,
+            'progress_percent', latest_percent,
+            'progress_message', latest_message,
+            'progress_entries', entry_count,
+            'created', NEW.created,
+            'updated', NEW.updated
+        );
+
+        PERFORM pg_notify('artifact_updated', payload::text);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Trigger on artifact table for updates
+CREATE TRIGGER artifact_updated_notify
+    AFTER UPDATE ON artifact
+    FOR EACH ROW
+    EXECUTE FUNCTION notify_artifact_updated();
+
+COMMENT ON FUNCTION notify_artifact_updated() IS 'Sends artifact update notifications via PostgreSQL LISTEN/NOTIFY (includes progress summary for progress-type artifacts)';

docker/distributable/migrations/20250101000009_timescaledb_history.sql

@@ -0,0 +1,616 @@
+-- Migration: TimescaleDB Entity History and Analytics
+-- Description: Creates append-only history hypertables for execution and worker tables.
+--              Uses JSONB diff format to track field-level changes via PostgreSQL triggers.
+--              Converts the event, enforcement, and execution tables into TimescaleDB
+--              hypertables (events are immutable; enforcements are updated exactly once;
+--              executions are updated ~4 times during their lifecycle).
+--              Includes continuous aggregates for dashboard analytics.
+--              See docs/plans/timescaledb-entity-history.md for full design.
+--
+--              NOTE: FK constraints that would reference hypertable targets were never
+--              created in earlier migrations (000004, 000005, 000006), so no DROP
+--              CONSTRAINT statements are needed here.
+-- Version: 20250101000009
+
+-- ============================================================================
+-- EXTENSION
+-- ============================================================================
+
+CREATE EXTENSION IF NOT EXISTS timescaledb;
+
+-- ============================================================================
+-- HELPER FUNCTIONS
+-- ============================================================================
+
+-- Returns a small {digest, size, type} object instead of the full JSONB value.
+-- Used in history triggers for columns that can be arbitrarily large (e.g. result).
+-- The full value is always available on the live row.
+CREATE OR REPLACE FUNCTION _jsonb_digest_summary(val JSONB)
+RETURNS JSONB AS $$
+BEGIN
+    IF val IS NULL THEN
+        RETURN NULL;
+    END IF;
+    RETURN jsonb_build_object(
+        'digest', 'md5:' || md5(val::text),
+        'size',   octet_length(val::text),
+        'type',   jsonb_typeof(val)
+    );
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION _jsonb_digest_summary(JSONB) IS
+    'Returns a compact {digest, size, type} summary of a JSONB value for use in history tables. '
+    'The digest is md5 of the text representation — sufficient for change-detection, not for security.';
+
+-- ============================================================================
+-- HISTORY TABLES
+-- ============================================================================
+
+-- ----------------------------------------------------------------------------
+-- execution_history
+-- ----------------------------------------------------------------------------
+
+CREATE TABLE execution_history (
+    time             TIMESTAMPTZ    NOT NULL DEFAULT NOW(),
+    operation        TEXT           NOT NULL,
+    entity_id        BIGINT         NOT NULL,
+    entity_ref       TEXT,
+    changed_fields   TEXT[]         NOT NULL DEFAULT '{}',
+    old_values       JSONB,
+    new_values       JSONB
+);
+
+SELECT create_hypertable('execution_history', 'time',
+    chunk_time_interval => INTERVAL '1 day');
+
+CREATE INDEX idx_execution_history_entity
+    ON execution_history (entity_id, time DESC);
+
+CREATE INDEX idx_execution_history_entity_ref
+    ON execution_history (entity_ref, time DESC);
+
+CREATE INDEX idx_execution_history_status_changes
+    ON execution_history (time DESC)
+    WHERE 'status' = ANY(changed_fields);
+
+CREATE INDEX idx_execution_history_changed_fields
+    ON execution_history USING GIN (changed_fields);
+
+COMMENT ON TABLE execution_history IS 'Append-only history of field-level changes to the execution table (TimescaleDB hypertable)';
+COMMENT ON COLUMN execution_history.time IS 'When the change occurred (hypertable partitioning dimension)';
+COMMENT ON COLUMN execution_history.operation IS 'INSERT, UPDATE, or DELETE';
+COMMENT ON COLUMN execution_history.entity_id IS 'execution.id of the changed row';
+COMMENT ON COLUMN execution_history.entity_ref IS 'Denormalized action_ref for JOIN-free queries';
+COMMENT ON COLUMN execution_history.changed_fields IS 'Array of field names that changed (empty for INSERT/DELETE)';
+COMMENT ON COLUMN execution_history.old_values IS 'Previous values of changed fields (NULL for INSERT)';
+COMMENT ON COLUMN execution_history.new_values IS 'New values of changed fields (NULL for DELETE)';
+
+-- ----------------------------------------------------------------------------
+-- worker_history
+-- ----------------------------------------------------------------------------
+
+CREATE TABLE worker_history (
+    time             TIMESTAMPTZ    NOT NULL DEFAULT NOW(),
+    operation        TEXT           NOT NULL,
+    entity_id        BIGINT         NOT NULL,
+    entity_ref       TEXT,
+    changed_fields   TEXT[]         NOT NULL DEFAULT '{}',
+    old_values       JSONB,
+    new_values       JSONB
+);
+
+SELECT create_hypertable('worker_history', 'time',
+    chunk_time_interval => INTERVAL '7 days');
+
+CREATE INDEX idx_worker_history_entity
+    ON worker_history (entity_id, time DESC);
+
+CREATE INDEX idx_worker_history_entity_ref
+    ON worker_history (entity_ref, time DESC);
+
+CREATE INDEX idx_worker_history_status_changes
+    ON worker_history (time DESC)
+    WHERE 'status' = ANY(changed_fields);
+
+CREATE INDEX idx_worker_history_changed_fields
+    ON worker_history USING GIN (changed_fields);
+
+COMMENT ON TABLE worker_history IS 'Append-only history of field-level changes to the worker table (TimescaleDB hypertable)';
+COMMENT ON COLUMN worker_history.entity_ref IS 'Denormalized worker name for JOIN-free queries';
+
+-- ============================================================================
+-- CONVERT EVENT TABLE TO HYPERTABLE
+-- ============================================================================
+-- Events are immutable after insert — they are never updated. Instead of
+-- maintaining a separate event_history table to track changes that never
+-- happen, we convert the event table itself into a TimescaleDB hypertable
+-- partitioned on `created`. This gives us automatic time-based partitioning,
+-- compression, and retention for free.
+--
+-- No FK constraints reference event(id) — enforcement.event was created as a
+-- plain BIGINT in migration 000004 (hypertables cannot be FK targets).
+-- ----------------------------------------------------------------------------
+
+-- Replace the single-column PK with a composite PK that includes the
+-- partitioning column (required by TimescaleDB).
+ALTER TABLE event DROP CONSTRAINT event_pkey;
+ALTER TABLE event ADD PRIMARY KEY (id, created);
+
+SELECT create_hypertable('event', 'created',
+    chunk_time_interval => INTERVAL '1 day',
+    migrate_data        => true);
+
+COMMENT ON TABLE event IS 'Events are instances of triggers firing (TimescaleDB hypertable partitioned on created)';
+
+-- ============================================================================
+-- CONVERT ENFORCEMENT TABLE TO HYPERTABLE
+-- ============================================================================
+-- Enforcements are created and then updated exactly once (status changes from
+-- `created` to `processed` or `disabled` within ~1 second). This single update
+-- happens well before the 7-day compression window, so UPDATE on uncompressed
+-- chunks works without issues.
+--
+-- No FK constraints reference enforcement(id) — execution.enforcement was
+-- created as a plain BIGINT in migration 000005.
+-- ----------------------------------------------------------------------------
+
+ALTER TABLE enforcement DROP CONSTRAINT enforcement_pkey;
+ALTER TABLE enforcement ADD PRIMARY KEY (id, created);
+
+SELECT create_hypertable('enforcement', 'created',
+    chunk_time_interval => INTERVAL '1 day',
+    migrate_data        => true);
+
+COMMENT ON TABLE enforcement IS 'Enforcements represent rule triggering by events (TimescaleDB hypertable partitioned on created)';
+
+-- ============================================================================
+-- CONVERT EXECUTION TABLE TO HYPERTABLE
+-- ============================================================================
+-- Executions are updated ~4 times during their lifecycle (requested → scheduled
+-- → running → completed/failed), completing within at most ~1 day — well before
+-- the 7-day compression window. The `updated` column and its BEFORE UPDATE
+-- trigger are preserved (used by timeout monitor and UI).
+--
+-- No FK constraints reference execution(id) — inquiry.execution,
+-- workflow_execution.execution, execution.parent, and execution.original_execution
+-- were all created as plain BIGINT columns in migrations 000005 and 000006.
+--
+-- The existing execution_history hypertable and its trigger are preserved —
+-- they track field-level diffs of each update, which remains valuable for
+-- a mutable table.
+-- ----------------------------------------------------------------------------
+
+ALTER TABLE execution DROP CONSTRAINT execution_pkey;
+ALTER TABLE execution ADD PRIMARY KEY (id, created);
+
+SELECT create_hypertable('execution', 'created',
+    chunk_time_interval => INTERVAL '1 day',
+    migrate_data        => true);
+
+COMMENT ON TABLE execution IS 'Executions represent action runs with workflow support (TimescaleDB hypertable partitioned on created). Updated ~4 times during lifecycle, completing within ~1 day (well before 7-day compression window).';
+
+-- ============================================================================
+-- TRIGGER FUNCTIONS
+-- ============================================================================
+
+-- ----------------------------------------------------------------------------
+-- execution history trigger
+-- Tracked fields: status, result, executor, worker, workflow_task, env_vars, started_at
+-- Note: result uses _jsonb_digest_summary() to avoid storing large payloads
+-- ----------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION record_execution_history()
+RETURNS TRIGGER AS $$
+DECLARE
+    changed TEXT[] := '{}';
+    old_vals JSONB := '{}';
+    new_vals JSONB := '{}';
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        INSERT INTO execution_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'INSERT', NEW.id, NEW.action_ref, '{}', NULL,
+                jsonb_build_object(
+                    'status', NEW.status,
+                    'action_ref', NEW.action_ref,
+                    'executor', NEW.executor,
+                    'worker', NEW.worker,
+                    'parent', NEW.parent,
+                    'enforcement', NEW.enforcement,
+                    'started_at', NEW.started_at
+                ));
+        RETURN NEW;
+    END IF;
+
+    IF TG_OP = 'DELETE' THEN
+        INSERT INTO execution_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'DELETE', OLD.id, OLD.action_ref, '{}', NULL, NULL);
+        RETURN OLD;
+    END IF;
+
+    -- UPDATE: detect which fields changed
+
+    IF OLD.status IS DISTINCT FROM NEW.status THEN
+        changed := array_append(changed, 'status');
+        old_vals := old_vals || jsonb_build_object('status', OLD.status);
+        new_vals := new_vals || jsonb_build_object('status', NEW.status);
+    END IF;
+
+    -- Result: store a compact digest instead of the full JSONB to avoid bloat.
+    -- The live execution row always has the complete result.
+    IF OLD.result IS DISTINCT FROM NEW.result THEN
+        changed := array_append(changed, 'result');
+        old_vals := old_vals || jsonb_build_object('result', _jsonb_digest_summary(OLD.result));
+        new_vals := new_vals || jsonb_build_object('result', _jsonb_digest_summary(NEW.result));
+    END IF;
+
+    IF OLD.executor IS DISTINCT FROM NEW.executor THEN
+        changed := array_append(changed, 'executor');
+        old_vals := old_vals || jsonb_build_object('executor', OLD.executor);
+        new_vals := new_vals || jsonb_build_object('executor', NEW.executor);
+    END IF;
+
+    IF OLD.worker IS DISTINCT FROM NEW.worker THEN
+        changed := array_append(changed, 'worker');
+        old_vals := old_vals || jsonb_build_object('worker', OLD.worker);
+        new_vals := new_vals || jsonb_build_object('worker', NEW.worker);
+    END IF;
+
+    IF OLD.workflow_task IS DISTINCT FROM NEW.workflow_task THEN
+        changed := array_append(changed, 'workflow_task');
+        old_vals := old_vals || jsonb_build_object('workflow_task', OLD.workflow_task);
+        new_vals := new_vals || jsonb_build_object('workflow_task', NEW.workflow_task);
+    END IF;
+
+    IF OLD.env_vars IS DISTINCT FROM NEW.env_vars THEN
+        changed := array_append(changed, 'env_vars');
+        old_vals := old_vals || jsonb_build_object('env_vars', OLD.env_vars);
+        new_vals := new_vals || jsonb_build_object('env_vars', NEW.env_vars);
+    END IF;
+
+    IF OLD.started_at IS DISTINCT FROM NEW.started_at THEN
+        changed := array_append(changed, 'started_at');
+        old_vals := old_vals || jsonb_build_object('started_at', OLD.started_at);
+        new_vals := new_vals || jsonb_build_object('started_at', NEW.started_at);
+    END IF;
+
+    -- Only record if something actually changed
+    IF array_length(changed, 1) > 0 THEN
+        INSERT INTO execution_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'UPDATE', NEW.id, NEW.action_ref, changed, old_vals, new_vals);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION record_execution_history() IS 'Records field-level changes to execution table in execution_history hypertable';
+
+-- ----------------------------------------------------------------------------
+-- worker history trigger
+-- Tracked fields: name, status, capabilities, meta, host, port
+-- Excludes: last_heartbeat when it is the only field that changed
+-- ----------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION record_worker_history()
+RETURNS TRIGGER AS $$
+DECLARE
+    changed TEXT[] := '{}';
+    old_vals JSONB := '{}';
+    new_vals JSONB := '{}';
+BEGIN
+    IF TG_OP = 'INSERT' THEN
+        INSERT INTO worker_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'INSERT', NEW.id, NEW.name, '{}', NULL,
+                jsonb_build_object(
+                    'name', NEW.name,
+                    'worker_type', NEW.worker_type,
+                    'worker_role', NEW.worker_role,
+                    'status', NEW.status,
+                    'host', NEW.host,
+                    'port', NEW.port
+                ));
+        RETURN NEW;
+    END IF;
+
+    IF TG_OP = 'DELETE' THEN
+        INSERT INTO worker_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'DELETE', OLD.id, OLD.name, '{}', NULL, NULL);
+        RETURN OLD;
+    END IF;
+
+    -- UPDATE: detect which fields changed
+    IF OLD.name IS DISTINCT FROM NEW.name THEN
+        changed := array_append(changed, 'name');
+        old_vals := old_vals || jsonb_build_object('name', OLD.name);
+        new_vals := new_vals || jsonb_build_object('name', NEW.name);
+    END IF;
+
+    IF OLD.status IS DISTINCT FROM NEW.status THEN
+        changed := array_append(changed, 'status');
+        old_vals := old_vals || jsonb_build_object('status', OLD.status);
+        new_vals := new_vals || jsonb_build_object('status', NEW.status);
+    END IF;
+
+    IF OLD.capabilities IS DISTINCT FROM NEW.capabilities THEN
+        changed := array_append(changed, 'capabilities');
+        old_vals := old_vals || jsonb_build_object('capabilities', OLD.capabilities);
+        new_vals := new_vals || jsonb_build_object('capabilities', NEW.capabilities);
+    END IF;
+
+    IF OLD.meta IS DISTINCT FROM NEW.meta THEN
+        changed := array_append(changed, 'meta');
+        old_vals := old_vals || jsonb_build_object('meta', OLD.meta);
+        new_vals := new_vals || jsonb_build_object('meta', NEW.meta);
+    END IF;
+
+    IF OLD.host IS DISTINCT FROM NEW.host THEN
+        changed := array_append(changed, 'host');
+        old_vals := old_vals || jsonb_build_object('host', OLD.host);
+        new_vals := new_vals || jsonb_build_object('host', NEW.host);
+    END IF;
+
+    IF OLD.port IS DISTINCT FROM NEW.port THEN
+        changed := array_append(changed, 'port');
+        old_vals := old_vals || jsonb_build_object('port', OLD.port);
+        new_vals := new_vals || jsonb_build_object('port', NEW.port);
+    END IF;
+
+    -- Only record if something besides last_heartbeat changed.
+    -- Pure heartbeat-only updates are excluded to avoid high-volume noise.
+    IF array_length(changed, 1) > 0 THEN
+        INSERT INTO worker_history (time, operation, entity_id, entity_ref, changed_fields, old_values, new_values)
+        VALUES (NOW(), 'UPDATE', NEW.id, NEW.name, changed, old_vals, new_vals);
+    END IF;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION record_worker_history() IS 'Records field-level changes to worker table in worker_history hypertable. Excludes heartbeat-only updates.';
+
+-- ============================================================================
+-- ATTACH TRIGGERS TO OPERATIONAL TABLES
+-- ============================================================================
+
+CREATE TRIGGER execution_history_trigger
+    AFTER INSERT OR UPDATE OR DELETE ON execution
+    FOR EACH ROW
+    EXECUTE FUNCTION record_execution_history();
+
+CREATE TRIGGER worker_history_trigger
+    AFTER INSERT OR UPDATE OR DELETE ON worker
+    FOR EACH ROW
+    EXECUTE FUNCTION record_worker_history();
+
+-- ============================================================================
+-- COMPRESSION POLICIES
+-- ============================================================================
+
+-- History tables
+ALTER TABLE execution_history SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'entity_id',
+    timescaledb.compress_orderby = 'time DESC'
+);
+SELECT add_compression_policy('execution_history', INTERVAL '7 days');
+
+ALTER TABLE worker_history SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'entity_id',
+    timescaledb.compress_orderby = 'time DESC'
+);
+SELECT add_compression_policy('worker_history', INTERVAL '7 days');
+
+-- Event table (hypertable)
+ALTER TABLE event SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'trigger_ref',
+    timescaledb.compress_orderby = 'created DESC'
+);
+SELECT add_compression_policy('event', INTERVAL '7 days');
+
+-- Enforcement table (hypertable)
+ALTER TABLE enforcement SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'rule_ref',
+    timescaledb.compress_orderby = 'created DESC'
+);
+SELECT add_compression_policy('enforcement', INTERVAL '7 days');
+
+-- Execution table (hypertable)
+ALTER TABLE execution SET (
+    timescaledb.compress,
+    timescaledb.compress_segmentby = 'action_ref',
+    timescaledb.compress_orderby = 'created DESC'
+);
+SELECT add_compression_policy('execution', INTERVAL '7 days');
+
+-- ============================================================================
+-- RETENTION POLICIES
+-- ============================================================================
+
+SELECT add_retention_policy('execution_history', INTERVAL '90 days');
+SELECT add_retention_policy('worker_history', INTERVAL '180 days');
+SELECT add_retention_policy('event', INTERVAL '90 days');
+SELECT add_retention_policy('enforcement', INTERVAL '90 days');
+SELECT add_retention_policy('execution', INTERVAL '90 days');
+
+-- ============================================================================
+-- CONTINUOUS AGGREGATES
+-- ============================================================================
+
+-- Drop existing continuous aggregates if they exist, so this migration can be
+-- re-run safely after a partial failure. (TimescaleDB continuous aggregates
+-- must be dropped with CASCADE to remove their associated policies.)
+DROP MATERIALIZED VIEW IF EXISTS execution_status_hourly CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS execution_throughput_hourly CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS event_volume_hourly CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS worker_status_hourly CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS enforcement_volume_hourly CASCADE;
+DROP MATERIALIZED VIEW IF EXISTS execution_volume_hourly CASCADE;
+
+-- ----------------------------------------------------------------------------
+-- execution_status_hourly
+-- Tracks execution status transitions per hour, grouped by action_ref and new status.
+-- Powers: execution throughput chart, failure rate widget, status breakdown over time.
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW execution_status_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', time) AS bucket,
+    entity_ref AS action_ref,
+    new_values->>'status' AS new_status,
+    COUNT(*) AS transition_count
+FROM execution_history
+WHERE 'status' = ANY(changed_fields)
+GROUP BY bucket, entity_ref, new_values->>'status'
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('execution_status_hourly',
+    start_offset    => INTERVAL '7 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '30 minutes'
+);
+
+-- ----------------------------------------------------------------------------
+-- execution_throughput_hourly
+-- Tracks total execution creation volume per hour, regardless of status.
+-- Powers: execution throughput sparkline on the dashboard.
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW execution_throughput_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', time) AS bucket,
+    entity_ref AS action_ref,
+    COUNT(*) AS execution_count
+FROM execution_history
+WHERE operation = 'INSERT'
+GROUP BY bucket, entity_ref
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('execution_throughput_hourly',
+    start_offset    => INTERVAL '7 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '30 minutes'
+);
+
+-- ----------------------------------------------------------------------------
+-- event_volume_hourly
+-- Tracks event creation volume per hour by trigger ref.
+-- Powers: event throughput monitoring widget.
+-- NOTE: Queries the event table directly (it is now a hypertable) instead of
+--       a separate event_history table.
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW event_volume_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', created) AS bucket,
+    trigger_ref,
+    COUNT(*) AS event_count
+FROM event
+GROUP BY bucket, trigger_ref
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('event_volume_hourly',
+    start_offset    => INTERVAL '7 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '30 minutes'
+);
+
+-- ----------------------------------------------------------------------------
+-- worker_status_hourly
+-- Tracks worker status changes per hour (online/offline/draining transitions).
+-- Powers: worker health trends widget.
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW worker_status_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', time) AS bucket,
+    entity_ref AS worker_name,
+    new_values->>'status' AS new_status,
+    COUNT(*) AS transition_count
+FROM worker_history
+WHERE 'status' = ANY(changed_fields)
+GROUP BY bucket, entity_ref, new_values->>'status'
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('worker_status_hourly',
+    start_offset    => INTERVAL '30 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '1 hour'
+);
+
+-- ----------------------------------------------------------------------------
+-- enforcement_volume_hourly
+-- Tracks enforcement creation volume per hour by rule ref.
+-- Powers: rule activation rate monitoring.
+-- NOTE: Queries the enforcement table directly (it is now a hypertable)
+--       instead of a separate enforcement_history table.
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW enforcement_volume_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', created) AS bucket,
+    rule_ref,
+    COUNT(*) AS enforcement_count
+FROM enforcement
+GROUP BY bucket, rule_ref
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('enforcement_volume_hourly',
+    start_offset    => INTERVAL '7 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '30 minutes'
+);
+
+-- ----------------------------------------------------------------------------
+-- execution_volume_hourly
+-- Tracks execution creation volume per hour by action_ref and status.
+-- This queries the execution hypertable directly (like event_volume_hourly
+-- queries the event table). Complements the existing execution_status_hourly
+-- and execution_throughput_hourly aggregates which query execution_history.
+--
+-- Use case: direct execution volume monitoring without relying on the history
+-- trigger (belt-and-suspenders, plus captures the initial status at creation).
+-- ----------------------------------------------------------------------------
+
+CREATE MATERIALIZED VIEW execution_volume_hourly
+WITH (timescaledb.continuous) AS
+SELECT
+    time_bucket('1 hour', created) AS bucket,
+    action_ref,
+    status AS initial_status,
+    COUNT(*) AS execution_count
+FROM execution
+GROUP BY bucket, action_ref, status
+WITH NO DATA;
+
+SELECT add_continuous_aggregate_policy('execution_volume_hourly',
+    start_offset    => INTERVAL '7 days',
+    end_offset      => INTERVAL '1 hour',
+    schedule_interval => INTERVAL '30 minutes'
+);
+
+-- ============================================================================
+-- INITIAL REFRESH NOTE
+-- ============================================================================
+-- NOTE: refresh_continuous_aggregate() cannot run inside a transaction block,
+-- and the migration runner wraps each file in BEGIN/COMMIT. The continuous
+-- aggregate policies configured above will automatically backfill data within
+-- their first scheduled interval (30 min – 1 hour). On a fresh database there
+-- is no history data to backfill anyway.
+--
+-- If you need an immediate manual refresh after migration, run outside a
+-- transaction:
+--   CALL refresh_continuous_aggregate('execution_status_hourly', NULL, NOW());
+--   CALL refresh_continuous_aggregate('execution_throughput_hourly', NULL, NOW());
+--   CALL refresh_continuous_aggregate('event_volume_hourly', NULL, NOW());
+--   CALL refresh_continuous_aggregate('worker_status_hourly', NULL, NOW());
+--   CALL refresh_continuous_aggregate('enforcement_volume_hourly', NULL, NOW());
+--   CALL refresh_continuous_aggregate('execution_volume_hourly', NULL, NOW());

docker/distributable/migrations/20250101000010_artifact_content.sql

@@ -0,0 +1,202 @@
+-- Migration: Artifact Content System
+-- Description: Enhances the artifact table with content fields (name, description,
+--              content_type, size_bytes, execution link, structured data, visibility)
+--              and creates the artifact_version table for versioned file/data storage.
+--
+--              The artifact table now serves as the "header" for a logical artifact,
+--              while artifact_version rows hold the actual immutable content snapshots.
+--              Progress-type artifacts store their live state directly in artifact.data
+--              (append-style updates without creating new versions).
+--
+-- Version: 20250101000010
+
+-- ============================================================================
+-- ENHANCE ARTIFACT TABLE
+-- ============================================================================
+
+-- Human-readable name (e.g. "Build Log", "Test Results")
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS name TEXT;
+
+-- Optional longer description
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS description TEXT;
+
+-- MIME content type (e.g. "application/json", "text/plain", "image/png")
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS content_type TEXT;
+
+-- Total size in bytes of the latest version's content (NULL for progress artifacts)
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS size_bytes BIGINT;
+
+-- Execution that produced/owns this artifact (plain BIGINT, no FK — execution is a hypertable)
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS execution BIGINT;
+
+-- Structured data for progress-type artifacts and small structured payloads.
+-- Progress artifacts append entries here; file artifacts may store parsed metadata.
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS data JSONB;
+
+-- Visibility: public artifacts are viewable by all authenticated users;
+-- private artifacts are restricted based on the artifact's scope/owner.
+-- The scope (identity, action, pack, etc.) + owner fields define who can access
+-- a private artifact. Full RBAC enforcement is deferred — for now the column
+-- enables filtering and is available for future permission checks.
+ALTER TABLE artifact ADD COLUMN IF NOT EXISTS visibility artifact_visibility_enum NOT NULL DEFAULT 'private';
+
+-- New indexes for the added columns
+CREATE INDEX IF NOT EXISTS idx_artifact_execution ON artifact(execution);
+CREATE INDEX IF NOT EXISTS idx_artifact_name ON artifact(name);
+CREATE INDEX IF NOT EXISTS idx_artifact_execution_type ON artifact(execution, type);
+CREATE INDEX IF NOT EXISTS idx_artifact_visibility ON artifact(visibility);
+CREATE INDEX IF NOT EXISTS idx_artifact_visibility_scope ON artifact(visibility, scope, owner);
+
+-- Comments for new columns
+COMMENT ON COLUMN artifact.name IS 'Human-readable artifact name';
+COMMENT ON COLUMN artifact.description IS 'Optional description of the artifact';
+COMMENT ON COLUMN artifact.content_type IS 'MIME content type (e.g. application/json, text/plain)';
+COMMENT ON COLUMN artifact.size_bytes IS 'Size of latest version content in bytes';
+COMMENT ON COLUMN artifact.execution IS 'Execution that produced this artifact (no FK — execution is a hypertable)';
+COMMENT ON COLUMN artifact.data IS 'Structured JSONB data for progress artifacts or metadata';
+COMMENT ON COLUMN artifact.visibility IS 'Access visibility: public (all users) or private (scope/owner-restricted)';
+
+
+-- ============================================================================
+-- ARTIFACT_VERSION TABLE
+-- ============================================================================
+-- Each row is an immutable snapshot of artifact content. File-type artifacts get
+-- a new version on each upload; progress-type artifacts do NOT use versions
+-- (they update artifact.data directly).
+
+CREATE TABLE artifact_version (
+    id BIGSERIAL PRIMARY KEY,
+
+    -- Parent artifact
+    artifact BIGINT NOT NULL REFERENCES artifact(id) ON DELETE CASCADE,
+
+    -- Monotonically increasing version number within the artifact (1-based)
+    version INTEGER NOT NULL,
+
+    -- MIME content type for this specific version (may differ from parent)
+    content_type TEXT,
+
+    -- Size of the content in bytes
+    size_bytes BIGINT,
+
+    -- Binary content (file uploads, DB-stored). NULL for file-backed versions.
+    content BYTEA,
+
+    -- Structured content (JSON payloads, parsed results, etc.)
+    content_json JSONB,
+
+    -- Relative path from artifacts_dir root for disk-stored content.
+    -- When set, content BYTEA is NULL — file lives on shared volume.
+    -- Pattern: {ref_slug}/v{version}.{ext}
+    -- e.g., "mypack/build_log/v1.txt"
+    file_path TEXT,
+
+    -- Free-form metadata about this version (e.g. commit hash, build number)
+    meta JSONB,
+
+    -- Who or what created this version (identity ref, action ref, "system", etc.)
+    created_by TEXT,
+
+    -- Immutable — no updated column
+    created TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Unique constraint: one version number per artifact
+ALTER TABLE artifact_version
+    ADD CONSTRAINT uq_artifact_version_artifact_version UNIQUE (artifact, version);
+
+-- Indexes
+CREATE INDEX idx_artifact_version_artifact ON artifact_version(artifact);
+CREATE INDEX idx_artifact_version_artifact_version ON artifact_version(artifact, version DESC);
+CREATE INDEX idx_artifact_version_created ON artifact_version(created DESC);
+CREATE INDEX idx_artifact_version_file_path ON artifact_version(file_path) WHERE file_path IS NOT NULL;
+
+-- Comments
+COMMENT ON TABLE artifact_version IS 'Immutable content snapshots for artifacts (file uploads, structured data)';
+COMMENT ON COLUMN artifact_version.artifact IS 'Parent artifact this version belongs to';
+COMMENT ON COLUMN artifact_version.version IS 'Version number (1-based, monotonically increasing per artifact)';
+COMMENT ON COLUMN artifact_version.content_type IS 'MIME content type for this version';
+COMMENT ON COLUMN artifact_version.size_bytes IS 'Size of content in bytes';
+COMMENT ON COLUMN artifact_version.content IS 'Binary content (file data)';
+COMMENT ON COLUMN artifact_version.content_json IS 'Structured JSON content';
+COMMENT ON COLUMN artifact_version.meta IS 'Free-form metadata about this version';
+COMMENT ON COLUMN artifact_version.created_by IS 'Who created this version (identity ref, action ref, system)';
+COMMENT ON COLUMN artifact_version.file_path IS 'Relative path from artifacts_dir root for disk-stored content. When set, content BYTEA is NULL — file lives on shared volume.';
+
+
+-- ============================================================================
+-- HELPER FUNCTION: next_artifact_version
+-- ============================================================================
+-- Returns the next version number for an artifact (MAX(version) + 1, or 1 if none).
+
+CREATE OR REPLACE FUNCTION next_artifact_version(p_artifact_id BIGINT)
+RETURNS INTEGER AS $$
+DECLARE
+    v_next INTEGER;
+BEGIN
+    SELECT COALESCE(MAX(version), 0) + 1
+    INTO v_next
+    FROM artifact_version
+    WHERE artifact = p_artifact_id;
+
+    RETURN v_next;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION next_artifact_version IS 'Returns the next version number for the given artifact';
+
+
+-- ============================================================================
+-- RETENTION ENFORCEMENT FUNCTION
+-- ============================================================================
+-- Called after inserting a new version to enforce the artifact retention policy.
+-- For 'versions' policy: deletes oldest versions beyond the limit.
+-- Time-based policies (days/hours/minutes) are handled by a scheduled job (not this trigger).
+
+CREATE OR REPLACE FUNCTION enforce_artifact_retention()
+RETURNS TRIGGER AS $$
+DECLARE
+    v_policy artifact_retention_enum;
+    v_limit INTEGER;
+    v_count INTEGER;
+BEGIN
+    SELECT retention_policy, retention_limit
+    INTO v_policy, v_limit
+    FROM artifact
+    WHERE id = NEW.artifact;
+
+    IF v_policy = 'versions' AND v_limit > 0 THEN
+        -- Count existing versions
+        SELECT COUNT(*) INTO v_count
+        FROM artifact_version
+        WHERE artifact = NEW.artifact;
+
+        -- If over limit, delete the oldest ones
+        IF v_count > v_limit THEN
+            DELETE FROM artifact_version
+            WHERE id IN (
+                SELECT id
+                FROM artifact_version
+                WHERE artifact = NEW.artifact
+                ORDER BY version ASC
+                LIMIT (v_count - v_limit)
+            );
+        END IF;
+    END IF;
+
+    -- Update parent artifact size_bytes with the new version's size
+    UPDATE artifact
+    SET size_bytes = NEW.size_bytes,
+        content_type = COALESCE(NEW.content_type, content_type)
+    WHERE id = NEW.artifact;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trg_enforce_artifact_retention
+    AFTER INSERT ON artifact_version
+    FOR EACH ROW
+    EXECUTE FUNCTION enforce_artifact_retention();
+
+COMMENT ON FUNCTION enforce_artifact_retention IS 'Enforces version-count retention policy and syncs size to parent artifact';

docker/distributable/migrations/20250101000011_key_value_jsonb.sql

@@ -0,0 +1,17 @@
+-- Migration: Convert key.value from TEXT to JSONB
+--
+-- This allows keys to store structured data (objects, arrays, numbers, booleans)
+-- in addition to plain strings. Existing string values are wrapped in JSON string
+-- literals so they remain valid and accessible.
+--
+-- Before: value TEXT NOT NULL  (e.g., 'my-secret-token')
+-- After:  value JSONB NOT NULL (e.g., '"my-secret-token"' or '{"user":"admin","pass":"s3cret"}')
+
+-- Step 1: Convert existing TEXT values to JSONB.
+-- to_jsonb(text) wraps a plain string as a JSON string literal, e.g.:
+--   'hello'  ->  '"hello"'
+-- This preserves all existing values perfectly — encrypted values (base64 strings)
+-- become JSON strings, and plain text values become JSON strings.
+ALTER TABLE key
+    ALTER COLUMN value TYPE JSONB
+    USING to_jsonb(value);

docker/distributable/migrations/README.md

@@ -0,0 +1,348 @@
+# Attune Database Migrations
+
+This directory contains SQL migrations for the Attune automation platform database schema.
+
+## Overview
+
+Migrations are numbered and executed in order. Each migration file is named with a timestamp prefix to ensure proper ordering:
+
+```
+YYYYMMDDHHMMSS_description.sql
+```
+
+## Migration Files
+
+The schema is organized into 5 logical migration files:
+
+| File | Description |
+|------|-------------|
+| `20250101000001_initial_setup.sql` | Creates schema, service role, all enum types, and shared functions |
+| `20250101000002_core_tables.sql` | Creates pack, runtime, worker, identity, permission_set, permission_assignment, policy, and key tables |
+| `20250101000003_event_system.sql` | Creates trigger, sensor, event, and enforcement tables |
+| `20250101000004_execution_system.sql` | Creates action, rule, execution, inquiry, workflow orchestration tables (workflow_definition, workflow_execution, workflow_task_execution), and workflow views |
+| `20250101000005_supporting_tables.sql` | Creates notification, artifact, and queue_stats tables with performance indexes |
+
+### Migration Dependencies
+
+The migrations must be run in order due to foreign key dependencies:
+
+1. **Initial Setup** - Foundation (schema, enums, functions)
+2. **Core Tables** - Base entities (pack, runtime, worker, identity, permissions, policy, key)
+3. **Event System** - Event monitoring (trigger, sensor, event, enforcement)
+4. **Execution System** - Action execution (action, rule, execution, inquiry)
+5. **Supporting Tables** - Auxiliary features (notification, artifact)
+
+## Running Migrations
+
+### Using SQLx CLI
+
+```bash
+# Install sqlx-cli if not already installed
+cargo install sqlx-cli --no-default-features --features postgres
+
+# Run all pending migrations
+sqlx migrate run
+
+# Check migration status
+sqlx migrate info
+
+# Revert last migration (if needed)
+sqlx migrate revert
+```
+
+### Manual Execution
+
+You can also run migrations manually using `psql`:
+
+```bash
+# Run all migrations in order
+for file in migrations/202501*.sql; do
+    psql -U postgres -d attune -f "$file"
+done
+```
+
+Or individually:
+
+```bash
+psql -U postgres -d attune -f migrations/20250101000001_initial_setup.sql
+psql -U postgres -d attune -f migrations/20250101000002_core_tables.sql
+# ... etc
+```
+
+## Database Setup
+
+### Prerequisites
+
+1. PostgreSQL 14 or later installed
+2. Create the database:
+
+```bash
+createdb attune
+```
+
+3. Set environment variable:
+
+```bash
+export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/attune"
+```
+
+### Initial Setup
+
+```bash
+# Navigate to workspace root
+cd /path/to/attune
+
+# Run migrations
+sqlx migrate run
+
+# Verify tables were created
+psql -U postgres -d attune -c "\dt attune.*"
+```
+
+## Schema Overview
+
+The Attune schema includes 22 tables organized into logical groups:
+
+### Core Tables (Migration 2)
+- **pack**: Automation component bundles
+- **runtime**: Execution environments (Python, Node.js, containers)
+- **worker**: Execution workers
+- **identity**: Users and service accounts
+- **permission_set**: Permission groups (like roles)
+- **permission_assignment**: Identity-permission links (many-to-many)
+- **policy**: Execution policies (rate limiting, concurrency)
+- **key**: Secure configuration and secrets storage
+
+### Event System (Migration 3)
+- **trigger**: Event type definitions
+- **sensor**: Event monitors that watch for triggers
+- **event**: Event instances (trigger firings)
+- **enforcement**: Rule activation instances
+
+### Execution System (Migration 4)
+- **action**: Executable operations (can be workflows)
+- **rule**: Trigger-to-action automation logic
+- **execution**: Action execution instances (supports workflows)
+- **inquiry**: Human-in-the-loop interactions (approvals, inputs)
+- **workflow_definition**: YAML-based workflow definitions (composable action graphs)
+- **workflow_execution**: Runtime state tracking for workflow executions
+- **workflow_task_execution**: Individual task executions within workflows
+
+### Supporting Tables (Migration 5)
+- **notification**: Real-time system notifications (uses PostgreSQL LISTEN/NOTIFY)
+- **artifact**: Execution outputs (files, logs, progress data)
+- **queue_stats**: Real-time execution queue statistics for FIFO ordering
+
+## Key Features
+
+### Automatic Timestamps
+All tables include `created` and `updated` timestamps that are automatically managed by the `update_updated_column()` trigger function.
+
+### Reference Preservation
+Tables use both ID foreign keys and `*_ref` text columns. The ref columns preserve string references even when the referenced entity is deleted, maintaining complete audit trails.
+
+### Soft Deletes
+Foreign keys strategically use:
+- `ON DELETE CASCADE` - For dependent data that should be removed
+- `ON DELETE SET NULL` - To preserve historical records while breaking the link
+
+### Validation Constraints
+- **Reference format validation** - Lowercase, specific patterns (e.g., `pack.name`)
+- **Semantic version validation** - For pack versions
+- **Ownership validation** - Custom trigger for key table ownership rules
+- **Range checks** - Port numbers, positive thresholds, etc.
+
+### Performance Optimization
+- **B-tree indexes** - On frequently queried columns (IDs, refs, status, timestamps)
+- **Partial indexes** - For filtered queries (e.g., `enabled = TRUE`)
+- **GIN indexes** - On JSONB and array columns for fast containment queries
+- **Composite indexes** - For common multi-column query patterns
+
+### PostgreSQL Features
+- **JSONB** - Flexible schema storage for configurations, payloads, results
+- **Array types** - Multi-value fields (tags, parameters, dependencies)
+- **Custom enum types** - Constrained string values with type safety
+- **Triggers** - Data validation, timestamp management, notifications
+- **pg_notify** - Real-time notifications via PostgreSQL's LISTEN/NOTIFY
+
+## Service Role
+
+The migrations create a `svc_attune` role with appropriate permissions. **Change the password in production:**
+
+```sql
+ALTER ROLE svc_attune WITH PASSWORD 'secure_password_here';
+```
+
+The default password is `attune_service_password` (only for development).
+
+## Rollback Strategy
+
+### Complete Reset
+
+To completely reset the database:
+
+```bash
+# Drop and recreate
+dropdb attune
+createdb attune
+sqlx migrate run
+```
+
+Or drop just the schema:
+
+```sql
+psql -U postgres -d attune -c "DROP SCHEMA attune CASCADE;"
+```
+
+Then re-run migrations.
+
+### Individual Migration Revert
+
+With SQLx CLI:
+
+```bash
+sqlx migrate revert
+```
+
+Or manually remove from tracking:
+
+```sql
+DELETE FROM _sqlx_migrations WHERE version = 20250101000001;
+```
+
+## Best Practices
+
+1. **Never edit existing migrations** - Create new migrations to modify schema
+2. **Test migrations** - Always test on a copy of production data first
+3. **Backup before migrating** - Backup production database before applying migrations
+4. **Review changes** - Review all migrations before applying to production
+5. **Version control** - Keep migrations in version control (they are!)
+6. **Document changes** - Add comments to complex migrations
+
+## Development Workflow
+
+1. Create new migration file with timestamp:
+   ```bash
+   touch migrations/$(date +%Y%m%d%H%M%S)_description.sql
+   ```
+
+2. Write migration SQL (follow existing patterns)
+
+3. Test migration:
+   ```bash
+   sqlx migrate run
+   ```
+
+4. Verify changes:
+   ```bash
+   psql -U postgres -d attune
+   \d+ attune.table_name
+   ```
+
+5. Commit to version control
+
+## Production Deployment
+
+1. **Backup** production database
+2. **Review** all pending migrations
+3. **Test** migrations on staging environment with production data copy
+4. **Schedule** maintenance window if needed
+5. **Apply** migrations:
+   ```bash
+   sqlx migrate run
+   ```
+6. **Verify** application functionality
+7. **Monitor** for errors in logs
+
+## Troubleshooting
+
+### Migration already applied
+
+If you need to re-run a migration:
+
+```bash
+# Remove from migration tracking (SQLx)
+psql -U postgres -d attune -c "DELETE FROM _sqlx_migrations WHERE version = 20250101000001;"
+
+# Then re-run
+sqlx migrate run
+```
+
+### Permission denied
+
+Ensure the PostgreSQL user has sufficient permissions:
+
+```sql
+GRANT ALL PRIVILEGES ON DATABASE attune TO postgres;
+GRANT ALL PRIVILEGES ON SCHEMA attune TO postgres;
+```
+
+### Connection refused
+
+Check PostgreSQL is running:
+
+```bash
+# Linux/macOS
+pg_ctl status
+sudo systemctl status postgresql
+
+# Check if listening
+psql -U postgres -c "SELECT version();"
+```
+
+### Foreign key constraint violations
+
+Ensure migrations run in correct order. The consolidated migrations handle forward references correctly:
+- Migration 2 creates tables with forward references (commented as such)
+- Migration 3 and 4 add the foreign key constraints back
+
+## Schema Diagram
+
+```
+┌─────────────┐
+│    pack     │◄──┐
+└─────────────┘   │
+       ▲          │
+       │          │
+┌──────┴──────────┴──────┐
+│ runtime │ trigger │ ... │  (Core entities reference pack)
+└─────────┴─────────┴─────┘
+       ▲          ▲
+       │          │
+┌──────┴──────┐  │
+│   sensor    │──┘  (Sensors reference both runtime and trigger)
+└─────────────┘
+       │
+       ▼
+┌─────────────┐     ┌──────────────┐
+│    event    │────►│ enforcement  │  (Events trigger enforcements)
+└─────────────┘     └──────────────┘
+                           │
+                           ▼
+                    ┌──────────────┐
+                    │  execution   │  (Enforcements create executions)
+                    └──────────────┘
+```
+
+## Workflow Orchestration
+
+Migration 4 includes comprehensive workflow orchestration support:
+- **workflow_definition**: Stores parsed YAML workflow definitions with tasks, variables, and transitions
+- **workflow_execution**: Tracks runtime state including current/completed/failed tasks and variables
+- **workflow_task_execution**: Individual task execution tracking with retry and timeout support
+- **Action table extensions**: `is_workflow` and `workflow_def` columns link actions to workflows
+- **Helper views**: Three views for querying workflow state (summary, task detail, action links)
+
+## Queue Statistics
+
+Migration 5 includes the queue_stats table for execution ordering:
+- Tracks per-action queue length, active executions, and concurrency limits
+- Enables FIFO queue management with database persistence
+- Supports monitoring and API visibility of execution queues
+
+## Additional Resources
+
+- [SQLx Documentation](https://github.com/launchbadge/sqlx)
+- [PostgreSQL Documentation](https://www.postgresql.org/docs/)
+- [Attune Architecture Documentation](../docs/architecture.md)
+- [Attune Data Model Documentation](../docs/data-model.md)