patches/neon-provider-ownership.patch
Rama main · réplica de lectura del repositorio original.
diff --git a/Cargo.lock b/Cargo.lock
index 5ea4912c43ef5f3a0685c8558281d1f4b0d43e53..55c2d3f590ddf510a58e01fac00db79875a4c159 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1345,6 +1345,7 @@ dependencies = [
"fail",
"flate2",
"futures",
+ "hex",
"hostname-validator",
"http 1.3.1",
"http-body-util",
@@ -1376,6 +1377,7 @@ dependencies = [
"serde",
"serde_json",
"serde_with",
+ "sha2",
"signal-hook",
"tar",
"thiserror 1.0.69",
@@ -4578,6 +4580,7 @@ dependencies = [
"serde_json",
"serde_path_to_error",
"serde_with",
+ "sha2",
"smallvec",
"storage_broker",
"strum",
@@ -5956,6 +5959,7 @@ dependencies = [
"serde_json",
"sync_wrapper 0.1.2",
"test-context",
+ "thiserror 1.0.69",
"tokio",
"tokio-stream",
"tokio-util",
@@ -7157,6 +7161,7 @@ dependencies = [
"scopeguard",
"serde",
"serde_json",
+ "sha2",
"strum",
"strum_macros",
"thiserror 1.0.69",
diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml
index 558760b0ad0db8340b8ef6d8a94f268f8353b8ba..37b3d5477792da92f2036ea6376acde95a68173b 100644
--- a/compute_tools/Cargo.toml
+++ b/compute_tools/Cargo.toml
@@ -26,6 +26,7 @@ clap.workspace = true
fail.workspace = true
flate2.workspace = true
futures.workspace = true
+hex.workspace = true
http.workspace = true
http-body-util.workspace = true
hostname-validator = "1.1"
@@ -35,7 +36,7 @@ indexmap.workspace = true
itertools.workspace = true
jsonwebtoken.workspace = true
metrics.workspace = true
-nix.workspace = true
+nix = { workspace = true, features = ["user"] }
notify.workspace = true
num_cpus.workspace = true
once_cell.workspace = true
@@ -51,6 +52,7 @@ scopeguard.workspace = true
serde.workspace = true
serde_with.workspace = true
serde_json.workspace = true
+sha2.workspace = true
signal-hook.workspace = true
tar.workspace = true
tower.workspace = true
diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs
index f383683ef8a1b2bd8f22619c0141721f01f7eab8..c23ef415f06d9dd5165f8822a20070bbd0239e0a 100644
--- a/compute_tools/src/bin/compute_ctl.rs
+++ b/compute_tools/src/bin/compute_ctl.rs
@@ -129,6 +129,10 @@ struct Cli {
#[arg(short = 'c', long)]
pub config: Option<OsString>,
+ /// Durable ownership record used by strict self-hosted control planes.
+ #[arg(long)]
+ pub ownership_state_path: Option<OsString>,
+
#[arg(short = 'i', long, group = "compute-id")]
pub compute_id: String,
@@ -275,6 +279,7 @@ fn main() -> Result<()> {
build_tag: BUILD_TAG.to_string(),
control_plane_uri: cli.control_plane_uri,
config_path_test_only: cli.config,
+ ownership_state_path: cli.ownership_state_path,
},
config,
)?;
diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs
index c0f0289e064fe2cad13bbc227bf35f8f48ce137d..5ed23951e8fd1cbf8a896538bed8755083b40129 100644
--- a/compute_tools/src/compute.rs
+++ b/compute_tools/src/compute.rs
@@ -52,6 +52,7 @@
use crate::lsn_lease::launch_lsn_lease_bg_task_for_static;
use crate::metrics::COMPUTE_CTL_UP;
use crate::monitor::launch_monitor;
+use crate::ownership::ComputeOwnership;
use crate::pg_helpers::*;
use crate::pgbouncer::*;
use crate::rsyslog::{
@@ -131,6 +132,9 @@ pub struct ComputeNodeParams {
pub build_tag: String,
pub control_plane_uri: Option<String>,
pub config_path_test_only: Option<OsString>,
+ /// Optional durable ownership record. When set, compute_ctl starts empty
+ /// and accepts only exact, ownership-token-bound configuration replays.
+ pub ownership_state_path: Option<OsString>,
}
type TaskHandle = Mutex<Option<JoinHandle<()>>>;
@@ -155,6 +159,7 @@ pub struct ComputeNode {
// key: ext_archive_name, value: started download time, download_completed?
pub ext_download_progress: RwLock<HashMap<String, (DateTime<Utc>, bool)>>,
pub compute_ctl_config: ComputeCtlConfig,
+ pub ownership: ComputeOwnership,
/// Handle to the extension stats collection task
extension_stats_task: TaskHandle,
@@ -567,7 +572,7 @@ pub fn to_env_var_list(self) -> Vec<(String, String)> {
}
impl ComputeNode {
- pub fn new(params: ComputeNodeParams, config: ComputeConfig) -> Result<Self> {
+ pub fn new(params: ComputeNodeParams, mut config: ComputeConfig) -> Result<Self> {
let connstr = params.connstr.as_str();
let mut conn_conf = postgres::config::Config::from_str(connstr)
.context("cannot build postgres config from connstr")?;
@@ -599,6 +604,12 @@ pub fn new(params: ComputeNodeParams, config: ComputeConfig) -> Result<Self> {
conn_conf.options(&options);
tokio_conn_conf.options(&options);
+ let ownership = ComputeOwnership::load(params.ownership_state_path.clone())?;
+ if ownership.enabled() {
+ // Strictly owned computes are always reattached by an exact /configure
+ // replay. Never trust a static startup template as active ownership.
+ config.spec = None;
+ }
let mut new_state = ComputeState::new();
if let Some(spec) = config.spec {
let pspec = ParsedSpec::try_from(spec).map_err(|msg| anyhow::anyhow!(msg))?;
@@ -617,6 +628,7 @@ pub fn new(params: ComputeNodeParams, config: ComputeConfig) -> Result<Self> {
state_changed: Condvar::new(),
ext_download_progress: RwLock::new(HashMap::new()),
compute_ctl_config: config.compute_ctl_config,
+ ownership,
extension_stats_task: Mutex::new(None),
lfc_offload_task: Mutex::new(None),
})
@@ -829,18 +841,32 @@ fn start_compute(self: &Arc<Self>, pg_handle: &mut Option<PostgresHandle>) -> Re
}
let pspec = compute_state.pspec.as_ref().expect("spec must be set");
- info!(
- "starting compute for project {}, operation {}, tenant {}, timeline {}, project {}, branch {}, endpoint {}, features {:?}, spec.remote_extensions {:?}",
- pspec.spec.cluster.cluster_id.as_deref().unwrap_or("None"),
- pspec.spec.operation_uuid.as_deref().unwrap_or("None"),
- pspec.tenant_id,
- pspec.timeline_id,
- pspec.spec.project_id.as_deref().unwrap_or("None"),
- pspec.spec.branch_id.as_deref().unwrap_or("None"),
- pspec.spec.endpoint_id.as_deref().unwrap_or("None"),
- pspec.spec.features,
- pspec.spec.remote_extensions,
- );
+ if self.ownership.enabled() {
+ info!(
+ "starting owned compute for project {}, tenant {}, timeline {}, project {}, branch {}, endpoint {}, features {:?}, spec.remote_extensions {:?}",
+ pspec.spec.cluster.cluster_id.as_deref().unwrap_or("None"),
+ pspec.tenant_id,
+ pspec.timeline_id,
+ pspec.spec.project_id.as_deref().unwrap_or("None"),
+ pspec.spec.branch_id.as_deref().unwrap_or("None"),
+ pspec.spec.endpoint_id.as_deref().unwrap_or("None"),
+ pspec.spec.features,
+ pspec.spec.remote_extensions,
+ );
+ } else {
+ info!(
+ "starting compute for project {}, operation {}, tenant {}, timeline {}, project {}, branch {}, endpoint {}, features {:?}, spec.remote_extensions {:?}",
+ pspec.spec.cluster.cluster_id.as_deref().unwrap_or("None"),
+ pspec.spec.operation_uuid.as_deref().unwrap_or("None"),
+ pspec.tenant_id,
+ pspec.timeline_id,
+ pspec.spec.project_id.as_deref().unwrap_or("None"),
+ pspec.spec.branch_id.as_deref().unwrap_or("None"),
+ pspec.spec.endpoint_id.as_deref().unwrap_or("None"),
+ pspec.spec.features,
+ pspec.spec.remote_extensions,
+ );
+ }
////// PRE-STARTUP PHASE: things that need to be finished before we start the Postgres process
@@ -2165,12 +2191,16 @@ pub fn reconfigure(&self) -> Result<()> {
self.pg_reload_conf()?;
}
- let unknown_op = "unknown".to_string();
- let op_id = spec.operation_uuid.as_ref().unwrap_or(&unknown_op);
- info!(
- "finished reconfiguration of compute node for operation {}",
- op_id
- );
+ if self.ownership.enabled() {
+ info!("finished reconfiguration of owned compute node");
+ } else {
+ let unknown_op = "unknown".to_string();
+ let op_id = spec.operation_uuid.as_ref().unwrap_or(&unknown_op);
+ info!(
+ "finished reconfiguration of compute node for operation {}",
+ op_id
+ );
+ }
Ok(())
}
diff --git a/compute_tools/src/compute_promote.rs b/compute_tools/src/compute_promote.rs
index 15b5fcfb4675fb15b0637e3a0af9db4ba2cad3de..4c933678c419801ec1402d3232f5c40eb41de8c3 100644
--- a/compute_tools/src/compute_promote.rs
+++ b/compute_tools/src/compute_promote.rs
@@ -152,7 +152,11 @@ async fn promote_impl(&self, cfg: PromoteConfig) -> anyhow::Result<PromoteState>
new_conf.push_str(&format!("port={}\n", set["port"]));
}
- tracing::debug!("applied spec: {:#?}", new_pspec.spec);
+ if self.params.ownership_state_path.is_some() {
+ tracing::debug!("applied owned compute spec; sensitive fields omitted");
+ } else {
+ tracing::debug!("applied spec: {:#?}", new_pspec.spec);
+ }
if self.params.lakebase_mode {
ComputeNode::set_spec(&self.params, &mut state, new_pspec);
} else {
diff --git a/compute_tools/src/configurator.rs b/compute_tools/src/configurator.rs
index 79eb80c4a07b15f160f2aad5a1f957a01ffbeea3..d749b9bfb9d14fc924b6fe705f0ad0e0f3504449 100644
--- a/compute_tools/src/configurator.rs
+++ b/compute_tools/src/configurator.rs
@@ -56,6 +56,13 @@ fn configurator_main_loop(compute: &Arc<ComputeNode>) {
compute.set_status(new_status);
} else if state.status == ComputeStatus::RefreshConfigurationPending {
+ if compute.ownership.enabled() {
+ error!(
+ "Refresh configuration: owned compute must use authenticated /configure replay"
+ );
+ state.set_status(ComputeStatus::Running, &compute.state_changed);
+ continue;
+ }
info!(
"compute node suspects its configuration is out of date, now refreshing configuration"
);
diff --git a/compute_tools/src/http/routes/configure.rs b/compute_tools/src/http/routes/configure.rs
index 943ff453578f38b5571c2d42af0f399be4d344b0..c06de8e3c28612365df3220c2ef0bcf3c4d28de5 100644
--- a/compute_tools/src/http/routes/configure.rs
+++ b/compute_tools/src/http/routes/configure.rs
@@ -1,9 +1,10 @@
use std::sync::Arc;
use axum::extract::State;
+use axum::http::HeaderMap;
use axum::response::Response;
use compute_api::requests::ConfigurationRequest;
-use compute_api::responses::{ComputeStatus, ComputeStatusResponse};
+use compute_api::responses::ComputeStatus;
use http::StatusCode;
use tokio::task;
use tracing::info;
@@ -11,6 +12,8 @@
use crate::compute::{ComputeNode, ParsedSpec};
use crate::http::JsonResponse;
use crate::http::extract::Json;
+use crate::http::routes::compute_status_response;
+use crate::ownership::OwnershipError;
// Accept spec in JSON format and request compute configuration. If anything
// goes wrong after we set the compute status to `ConfigurationPending` and
@@ -20,6 +23,7 @@
// case of errors.
pub(in crate::http) async fn configure(
State(compute): State<Arc<ComputeNode>>,
+ headers: HeaderMap,
request: Json<ConfigurationRequest>,
) -> Response {
let pspec = match ParsedSpec::try_from(request.0.spec) {
@@ -37,6 +41,9 @@ pub(in crate::http) async fn configure(
if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) {
return JsonResponse::invalid_status(state.status);
}
+ if let Err(error) = compute.ownership.authorize_configure(&headers, &pspec.spec) {
+ return ownership_error(error);
+ }
// Pass the tracing span to the main thread that performs the startup,
// so that the start_compute operation is considered a child of this
@@ -86,7 +93,20 @@ pub(in crate::http) async fn configure(
// Return current compute state if everything went well.
let state = compute.state.lock().unwrap().clone();
- let body = ComputeStatusResponse::from(&state);
+ let ownership_token = match compute.ownership.token() {
+ Ok(token) => token,
+ Err(error) => return ownership_error(error),
+ };
+ let body = compute_status_response(&state, ownership_token);
JsonResponse::success(StatusCode::OK, body)
}
+
+fn ownership_error(error: OwnershipError) -> Response {
+ match error {
+ OwnershipError::Conflict => JsonResponse::error(StatusCode::CONFLICT, error),
+ OwnershipError::Persistence(_) => {
+ JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error)
+ }
+ }
+}
diff --git a/compute_tools/src/http/routes/mod.rs b/compute_tools/src/http/routes/mod.rs
index c0f68701c6000ba2ab10d0e4e4141bf6b332b5b4..4ec7197d635db39b52a33e8c804dc05dd5a26fa8 100644
--- a/compute_tools/src/http/routes/mod.rs
+++ b/compute_tools/src/http/routes/mod.rs
@@ -20,21 +20,28 @@
pub(in crate::http) mod status;
pub(in crate::http) mod terminate;
-impl From<&ComputeState> for ComputeStatusResponse {
- fn from(state: &ComputeState) -> Self {
- ComputeStatusResponse {
- start_time: state.start_time,
- tenant: state
+pub(in crate::http) fn compute_status_response(
+ state: &ComputeState,
+ durable_operation_uuid: Option<String>,
+) -> ComputeStatusResponse {
+ ComputeStatusResponse {
+ start_time: state.start_time,
+ tenant: state
+ .pspec
+ .as_ref()
+ .map(|pspec| pspec.tenant_id.to_string()),
+ timeline: state
+ .pspec
+ .as_ref()
+ .map(|pspec| pspec.timeline_id.to_string()),
+ operation_uuid: durable_operation_uuid.or_else(|| {
+ state
.pspec
.as_ref()
- .map(|pspec| pspec.tenant_id.to_string()),
- timeline: state
- .pspec
- .as_ref()
- .map(|pspec| pspec.timeline_id.to_string()),
- status: state.status,
- last_active: state.last_active,
- error: state.error.clone(),
- }
+ .and_then(|pspec| pspec.spec.operation_uuid.clone())
+ }),
+ status: state.status,
+ last_active: state.last_active,
+ error: state.error.clone(),
}
}
diff --git a/compute_tools/src/http/routes/promote.rs b/compute_tools/src/http/routes/promote.rs
index 865de0da91886d610baf2a8aeb389627f5c34757..4cf534efe8773e07c4065b2d163085e10b64bf4c 100644
--- a/compute_tools/src/http/routes/promote.rs
+++ b/compute_tools/src/http/routes/promote.rs
@@ -1,10 +1,13 @@
use crate::http::JsonResponse;
+use crate::ownership::OwnershipError;
use axum::extract::Json;
+use axum::http::HeaderMap;
use compute_api::responses::PromoteConfig;
use http::StatusCode;
pub(in crate::http) async fn promote(
compute: axum::extract::State<std::sync::Arc<crate::compute::ComputeNode>>,
+ headers: HeaderMap,
Json(cfg): Json<PromoteConfig>,
) -> axum::response::Response {
// Return early at the cost of extra parsing spec
@@ -12,6 +15,17 @@ pub(in crate::http) async fn promote(
Ok(p) => p,
Err(e) => return JsonResponse::error(StatusCode::BAD_REQUEST, e),
};
+ if let Err(error) = compute
+ .ownership
+ .authorize_spec_replay(&headers, &pspec.spec)
+ {
+ return match error {
+ OwnershipError::Conflict => JsonResponse::error(StatusCode::CONFLICT, error),
+ OwnershipError::Persistence(_) => {
+ JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error)
+ }
+ };
+ }
let cfg = PromoteConfig {
spec: pspec.spec,
diff --git a/compute_tools/src/http/routes/refresh_configuration.rs b/compute_tools/src/http/routes/refresh_configuration.rs
index 9b2f95ca5a94e206cdf1e74d1fdb8c35e1001ae8..5e892ab15261714c5168de8f47b4dba17505c1b1 100644
--- a/compute_tools/src/http/routes/refresh_configuration.rs
+++ b/compute_tools/src/http/routes/refresh_configuration.rs
@@ -21,6 +21,12 @@
pub(in crate::http) async fn refresh_configuration(
State(compute): State<Arc<ComputeNode>>,
) -> Response {
+ if compute.ownership.enabled() {
+ return JsonResponse::error(
+ StatusCode::CONFLICT,
+ "owned compute configuration must be replayed through /configure",
+ );
+ }
POSTGRES_PAGESTREAM_REQUEST_ERRORS.inc();
match compute.signal_refresh_configuration().await {
Ok(_) => StatusCode::OK.into_response(),
diff --git a/compute_tools/src/http/routes/status.rs b/compute_tools/src/http/routes/status.rs
index 8ed1299d6bdb0509f668fb9b13e1093be93e4b73..e1e44d2d919a0e7e77d21825b4dbc03afb3c62e8 100644
--- a/compute_tools/src/http/routes/status.rs
+++ b/compute_tools/src/http/routes/status.rs
@@ -1,18 +1,28 @@
-use std::ops::Deref;
use std::sync::Arc;
+use crate::compute::ComputeNode;
+use crate::http::JsonResponse;
+use crate::http::routes::compute_status_response;
+use crate::ownership::OwnershipError;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::Response;
-use compute_api::responses::ComputeStatusResponse;
-
-use crate::compute::ComputeNode;
-use crate::http::JsonResponse;
/// Retrieve the state of the comute.
pub(in crate::http) async fn get_status(State(compute): State<Arc<ComputeNode>>) -> Response {
let state = compute.state.lock().unwrap();
- let body = ComputeStatusResponse::from(state.deref());
+ let ownership_token = match compute.ownership.token() {
+ Ok(token) => token,
+ Err(error) => {
+ return match error {
+ OwnershipError::Conflict => JsonResponse::error(StatusCode::CONFLICT, error),
+ OwnershipError::Persistence(_) => {
+ JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error)
+ }
+ };
+ }
+ };
+ let body = compute_status_response(&state, ownership_token);
JsonResponse::success(StatusCode::OK, body)
}
diff --git a/compute_tools/src/http/routes/terminate.rs b/compute_tools/src/http/routes/terminate.rs
index deac760f439ba30e472565de9b3285ef143706cf..24832a70384e7f6474f8e87b4c480303a17694b4 100644
--- a/compute_tools/src/http/routes/terminate.rs
+++ b/compute_tools/src/http/routes/terminate.rs
@@ -4,12 +4,15 @@
use axum::response::{IntoResponse, Response};
use axum_extra::extract::OptionalQuery;
use compute_api::responses::{ComputeStatus, TerminateMode, TerminateResponse};
+use http::HeaderMap;
use http::StatusCode;
use serde::Deserialize;
use std::sync::Arc;
use tokio::task;
use tracing::info;
+use crate::ownership::OwnershipError;
+
#[derive(Deserialize, Default)]
pub struct TerminateQuery {
mode: TerminateMode,
@@ -18,12 +21,21 @@ pub struct TerminateQuery {
/// Terminate the compute.
pub(in crate::http) async fn terminate(
State(compute): State<Arc<ComputeNode>>,
+ headers: HeaderMap,
OptionalQuery(terminate): OptionalQuery<TerminateQuery>,
) -> Response {
let mode = terminate.unwrap_or_default().mode;
{
let mut state = compute.state.lock().unwrap();
if state.status == ComputeStatus::Terminated {
+ if let Err(error) = compute.ownership.authorize_terminate(&headers) {
+ return match error {
+ OwnershipError::Conflict => JsonResponse::error(StatusCode::CONFLICT, error),
+ OwnershipError::Persistence(_) => {
+ JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error)
+ }
+ };
+ }
let response = TerminateResponse {
lsn: state.terminate_flush_lsn,
};
@@ -34,6 +46,17 @@ pub(in crate::http) async fn terminate(
return JsonResponse::invalid_status(state.status);
}
+ // Persist the ownership tombstone only after the request is otherwise
+ // eligible to mutate this compute, and before any process state changes.
+ if let Err(error) = compute.ownership.authorize_terminate(&headers) {
+ return match error {
+ OwnershipError::Conflict => JsonResponse::error(StatusCode::CONFLICT, error),
+ OwnershipError::Persistence(_) => {
+ JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error)
+ }
+ };
+ }
+
// If compute is Empty, there's no Postgres to terminate. The regular compute_ctl termination path
// assumes Postgres to be configured and running, so we just special-handle this case by exiting
// the process directly.
diff --git a/compute_tools/src/lib.rs b/compute_tools/src/lib.rs
index 85a6f955d95c5ca736fed43f87dcfa7fa9af5d36..08a4fcf114aa887d346a1e98f8a94cd98a7a2af1 100644
--- a/compute_tools/src/lib.rs
+++ b/compute_tools/src/lib.rs
@@ -23,6 +23,7 @@
pub mod metrics;
mod migration;
pub mod monitor;
+pub mod ownership;
pub mod params;
pub mod pg_helpers;
pub mod pg_isready;
diff --git a/compute_tools/src/ownership.rs b/compute_tools/src/ownership.rs
new file mode 100644
index 0000000000000000000000000000000000000000..e4f6ef53ded89912da806f595b22ebf06cfde58a
--- /dev/null
+++ b/compute_tools/src/ownership.rs
@@ -0,0 +1,710 @@
+use std::ffi::{OsStr, OsString};
+use std::fs::{self, File, Metadata, OpenOptions};
+use std::io::{ErrorKind, Read, Write};
+use std::path::{Path, PathBuf};
+use std::sync::{Mutex, MutexGuard};
+use std::thread;
+use std::time::{Duration, Instant};
+
+#[cfg(unix)]
+use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
+
+use anyhow::{Context, Result, anyhow, bail};
+use compute_api::spec::ComputeSpec;
+use http::HeaderMap;
+use nix::errno::Errno;
+use nix::fcntl::{Flock, FlockArg, OFlag};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+
+pub const HAKOPOD_OWNERSHIP_TOKEN_HEADER: &str = "hakopod-ownership-token";
+const OWNERSHIP_SCHEMA_VERSION: u8 = 1;
+const MAX_OWNERSHIP_RECORD_BYTES: u64 = 4096;
+#[cfg(not(test))]
+const OWNERSHIP_LOCK_TIMEOUT: Duration = Duration::from_secs(5);
+#[cfg(test)]
+const OWNERSHIP_LOCK_TIMEOUT: Duration = Duration::from_millis(250);
+const OWNERSHIP_LOCK_RETRY: Duration = Duration::from_millis(10);
+
+#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+enum OwnershipState {
+ Active,
+ Terminated,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+struct OwnershipRecord {
+ schema_version: u8,
+ token: String,
+ spec_sha256: String,
+ state: OwnershipState,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum OwnershipError {
+ #[error("compute ownership conflict")]
+ Conflict,
+ #[error("compute ownership persistence failed: {0}")]
+ Persistence(#[source] anyhow::Error),
+}
+
+pub struct ComputeOwnership {
+ path: Option<PathBuf>,
+ lock_path: Option<PathBuf>,
+ record: Mutex<Option<OwnershipRecord>>,
+ // Keep exclusive authority for the entire compute_ctl lifetime. A lock
+ // taken only around authorization leaves the following process mutation
+ // exposed to a second compute_ctl using the same durable record.
+ _lifetime_lock: Option<Flock<File>>,
+}
+
+impl ComputeOwnership {
+ pub fn load(path: Option<OsString>) -> Result<Self> {
+ let Some(path) = path.map(PathBuf::from) else {
+ return Ok(Self {
+ path: None,
+ lock_path: None,
+ record: Mutex::new(None),
+ _lifetime_lock: None,
+ });
+ };
+ if !path.is_absolute() || path.file_name().is_none() {
+ bail!("compute ownership state path must be an absolute file path");
+ }
+ let parent = path
+ .parent()
+ .ok_or_else(|| anyhow!("compute ownership state has no parent"))?;
+ validate_private_parent(parent)?;
+ let lock_path = parent
+ .join(lock_file_name(path.file_name().ok_or_else(|| {
+ anyhow!("compute ownership state has no file name")
+ })?));
+ let mut ownership = Self {
+ path: Some(path),
+ lock_path: Some(lock_path),
+ record: Mutex::new(None),
+ _lifetime_lock: None,
+ };
+ let lifetime_lock = ownership.acquire_file_lock()?;
+ ownership.record = Mutex::new(ownership.read_record()?);
+ ownership._lifetime_lock = Some(lifetime_lock);
+ Ok(ownership)
+ }
+
+ pub fn enabled(&self) -> bool {
+ self.path.is_some()
+ }
+
+ pub fn token(&self) -> std::result::Result<Option<String>, OwnershipError> {
+ if !self.enabled() {
+ return Ok(None);
+ }
+ let mut guard = self.lock_record().map_err(OwnershipError::Persistence)?;
+ let record = self.read_record().map_err(OwnershipError::Persistence)?;
+ *guard = record.clone();
+ Ok(record.map(|record| record.token))
+ }
+
+ pub fn authorize_configure(
+ &self,
+ headers: &HeaderMap,
+ spec: &ComputeSpec,
+ ) -> std::result::Result<(), OwnershipError> {
+ if !self.enabled() {
+ return Ok(());
+ }
+ let token = required_token(headers)?;
+ if spec.operation_uuid.as_deref() != Some(token) {
+ return Err(OwnershipError::Conflict);
+ }
+ let hash = spec_hash(spec).map_err(OwnershipError::Persistence)?;
+ let mut guard = self.lock_record().map_err(OwnershipError::Persistence)?;
+ let record = self.read_record().map_err(OwnershipError::Persistence)?;
+ *guard = record.clone();
+ match record {
+ Some(record)
+ if record.state == OwnershipState::Active
+ && record.token == token
+ && record.spec_sha256 == hash =>
+ {
+ Ok(())
+ }
+ Some(_) => Err(OwnershipError::Conflict),
+ None => {
+ let record = OwnershipRecord {
+ schema_version: OWNERSHIP_SCHEMA_VERSION,
+ token: token.to_owned(),
+ spec_sha256: hash,
+ state: OwnershipState::Active,
+ };
+ self.persist_new(&record)
+ .map_err(OwnershipError::Persistence)?;
+ *guard = Some(record);
+ Ok(())
+ }
+ }
+ }
+
+ pub fn authorize_spec_replay(
+ &self,
+ headers: &HeaderMap,
+ spec: &ComputeSpec,
+ ) -> std::result::Result<(), OwnershipError> {
+ if !self.enabled() {
+ return Ok(());
+ }
+ let token = required_token(headers)?;
+ if spec.operation_uuid.as_deref() != Some(token) {
+ return Err(OwnershipError::Conflict);
+ }
+ let hash = spec_hash(spec).map_err(OwnershipError::Persistence)?;
+ let mut guard = self.lock_record().map_err(OwnershipError::Persistence)?;
+ let record = self.read_record().map_err(OwnershipError::Persistence)?;
+ *guard = record.clone();
+ match record {
+ Some(record)
+ if record.state == OwnershipState::Active
+ && record.token == token
+ && record.spec_sha256 == hash =>
+ {
+ Ok(())
+ }
+ _ => Err(OwnershipError::Conflict),
+ }
+ }
+
+ pub fn authorize_terminate(
+ &self,
+ headers: &HeaderMap,
+ ) -> std::result::Result<(), OwnershipError> {
+ if !self.enabled() {
+ return Ok(());
+ }
+ let token = required_token(headers)?;
+ let mut guard = self.lock_record().map_err(OwnershipError::Persistence)?;
+ let record = self.read_record().map_err(OwnershipError::Persistence)?;
+ *guard = record.clone();
+ let Some(record) = record else {
+ return Err(OwnershipError::Conflict);
+ };
+ if record.token != token {
+ return Err(OwnershipError::Conflict);
+ }
+ if record.state == OwnershipState::Active {
+ let mut tombstone = record;
+ tombstone.state = OwnershipState::Terminated;
+ self.persist_replacement(&tombstone)
+ .map_err(OwnershipError::Persistence)?;
+ *guard = Some(tombstone);
+ }
+ Ok(())
+ }
+
+ fn lock_record(&self) -> Result<MutexGuard<'_, Option<OwnershipRecord>>> {
+ self.record
+ .lock()
+ .map_err(|_| anyhow!("compute ownership process lock is poisoned"))
+ }
+
+ fn acquire_file_lock(&self) -> Result<Flock<File>> {
+ let lock_path = self
+ .lock_path
+ .as_ref()
+ .ok_or_else(|| anyhow!("compute ownership lock path is unavailable"))?;
+ let parent = lock_path
+ .parent()
+ .ok_or_else(|| anyhow!("compute ownership lock has no parent"))?;
+ validate_private_parent(parent)?;
+ let deadline = Instant::now() + OWNERSHIP_LOCK_TIMEOUT;
+ loop {
+ let mut options = OpenOptions::new();
+ options.read(true).write(true).create(true);
+ #[cfg(unix)]
+ {
+ options
+ .mode(0o600)
+ .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK).bits());
+ }
+ let file = options
+ .open(lock_path)
+ .context("open compute ownership lock")?;
+ let metadata = file
+ .metadata()
+ .context("read compute ownership lock metadata")?;
+ validate_private_regular_file(&metadata, "compute ownership lock")?;
+ match Flock::lock(file, FlockArg::LockExclusiveNonblock) {
+ Ok(file_lock) => {
+ let path_metadata = fs::symlink_metadata(lock_path)
+ .context("inspect compute ownership lock path")?;
+ validate_private_regular_file(&path_metadata, "compute ownership lock path")?;
+ if same_file(&metadata, &path_metadata) {
+ return Ok(file_lock);
+ }
+ }
+ Err((_, error)) if error == Errno::EAGAIN => {}
+ Err((_, error)) => return Err(error).context("lock compute ownership state"),
+ }
+ if Instant::now() >= deadline {
+ bail!("timed out locking compute ownership state");
+ }
+ thread::sleep(OWNERSHIP_LOCK_RETRY);
+ }
+ }
+
+ fn read_record(&self) -> Result<Option<OwnershipRecord>> {
+ let path = self
+ .path
+ .as_ref()
+ .ok_or_else(|| anyhow!("compute ownership state path is unavailable"))?;
+ let initial_metadata = match fs::symlink_metadata(path) {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
+ Err(error) => return Err(error).context("inspect compute ownership state path"),
+ };
+ validate_private_regular_file(&initial_metadata, "compute ownership state path")?;
+ let mut options = OpenOptions::new();
+ options.read(true);
+ #[cfg(unix)]
+ {
+ options.custom_flags((OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK).bits());
+ }
+ let file = options.open(path).context("open compute ownership state")?;
+ let metadata = file.metadata().context("read compute ownership metadata")?;
+ validate_private_regular_file(&metadata, "compute ownership state")?;
+ if metadata.len() > MAX_OWNERSHIP_RECORD_BYTES {
+ bail!("compute ownership state exceeds its size bound");
+ }
+ let path_metadata =
+ fs::symlink_metadata(path).context("inspect compute ownership state path")?;
+ validate_private_regular_file(&path_metadata, "compute ownership state path")?;
+ if !same_file(&metadata, &path_metadata) {
+ bail!("compute ownership state changed while opening");
+ }
+ let record: OwnershipRecord =
+ serde_json::from_reader(file.take(MAX_OWNERSHIP_RECORD_BYTES + 1))
+ .context("decode compute ownership state")?;
+ validate_record(&record)?;
+ Ok(Some(record))
+ }
+
+ fn persist_new(&self, record: &OwnershipRecord) -> Result<()> {
+ let (path, parent, temp_path, file) = self.write_temp_record(record)?;
+ file.sync_all().context("sync compute ownership state")?;
+ match fs::hard_link(&temp_path, path) {
+ Ok(()) => {}
+ Err(error) if error.kind() == ErrorKind::AlreadyExists => {
+ let _ = fs::remove_file(&temp_path);
+ bail!("compute ownership state was concurrently created");
+ }
+ Err(error) => {
+ let _ = fs::remove_file(&temp_path);
+ return Err(error).context("publish new compute ownership state");
+ }
+ }
+ let publish_result = sync_directory(parent)
+ .and_then(|()| {
+ fs::remove_file(&temp_path).context("remove compute ownership state temporary file")
+ })
+ .and_then(|()| sync_directory(parent));
+ if publish_result.is_err() {
+ let _ = fs::remove_file(&temp_path);
+ }
+ publish_result
+ }
+
+ fn persist_replacement(&self, record: &OwnershipRecord) -> Result<()> {
+ let (path, parent, temp_path, file) = self.write_temp_record(record)?;
+ file.sync_all().context("sync compute ownership state")?;
+ let publish_result = fs::rename(&temp_path, path)
+ .context("publish replacement compute ownership state")
+ .and_then(|()| sync_directory(parent));
+ if publish_result.is_err() {
+ let _ = fs::remove_file(&temp_path);
+ }
+ publish_result
+ }
+
+ fn write_temp_record(&self, record: &OwnershipRecord) -> Result<(&Path, &Path, PathBuf, File)> {
+ let path = self
+ .path
+ .as_deref()
+ .ok_or_else(|| anyhow!("compute ownership state path is unavailable"))?;
+ let parent = path
+ .parent()
+ .ok_or_else(|| anyhow!("compute ownership state has no parent"))?;
+ validate_private_parent(parent)?;
+ let bytes = serde_json::to_vec(record).context("encode compute ownership state")?;
+ if bytes.len() as u64 > MAX_OWNERSHIP_RECORD_BYTES {
+ bail!("compute ownership state exceeds its size bound");
+ }
+ let temp_path = parent
+ .join(temp_file_name(path.file_name().ok_or_else(|| {
+ anyhow!("compute ownership state has no file name")
+ })?));
+ let mut options = OpenOptions::new();
+ options.write(true).create_new(true);
+ #[cfg(unix)]
+ {
+ options.mode(0o600).custom_flags(OFlag::O_NOFOLLOW.bits());
+ }
+ let mut file = options
+ .open(&temp_path)
+ .context("create compute ownership state temporary file")?;
+ let metadata = file
+ .metadata()
+ .context("read compute ownership temporary file metadata")?;
+ validate_private_regular_file(&metadata, "compute ownership temporary file")?;
+ if let Err(error) = file
+ .write_all(&bytes)
+ .context("write compute ownership state")
+ {
+ let _ = fs::remove_file(&temp_path);
+ return Err(error);
+ }
+ Ok((path, parent, temp_path, file))
+ }
+}
+
+fn required_token(headers: &HeaderMap) -> std::result::Result<&str, OwnershipError> {
+ let token = headers
+ .get(HAKOPOD_OWNERSHIP_TOKEN_HEADER)
+ .and_then(|value| value.to_str().ok())
+ .ok_or(OwnershipError::Conflict)?;
+ let parsed = Uuid::parse_str(token).map_err(|_| OwnershipError::Conflict)?;
+ if parsed.is_nil() || parsed.simple().to_string() != token {
+ return Err(OwnershipError::Conflict);
+ }
+ Ok(token)
+}
+
+fn validate_record(record: &OwnershipRecord) -> Result<()> {
+ if record.schema_version != OWNERSHIP_SCHEMA_VERSION {
+ bail!("unsupported compute ownership state version");
+ }
+ let token = Uuid::parse_str(&record.token).context("invalid compute ownership token")?;
+ if token.is_nil() || token.simple().to_string() != record.token {
+ bail!("invalid compute ownership token encoding");
+ }
+ if record.spec_sha256.len() != 64
+ || !record
+ .spec_sha256
+ .bytes()
+ .all(|value| value.is_ascii_digit() || (b'a'..=b'f').contains(&value))
+ {
+ bail!("invalid compute ownership spec hash");
+ }
+ Ok(())
+}
+
+fn validate_private_parent(parent: &Path) -> Result<()> {
+ let metadata = fs::symlink_metadata(parent).context("inspect compute ownership parent")?;
+ if metadata.file_type().is_symlink() || !metadata.is_dir() {
+ bail!("compute ownership parent must be a real directory");
+ }
+ #[cfg(unix)]
+ {
+ if metadata.uid() != effective_uid() {
+ bail!("compute ownership parent must be owned by the compute user");
+ }
+ if metadata.permissions().mode() & 0o077 != 0 {
+ bail!("compute ownership parent must not be accessible by group or other users");
+ }
+ }
+ Ok(())
+}
+
+fn validate_private_regular_file(metadata: &Metadata, description: &str) -> Result<()> {
+ if !metadata.is_file() {
+ bail!("{description} must be a regular file");
+ }
+ #[cfg(unix)]
+ {
+ if metadata.uid() != effective_uid() {
+ bail!("{description} must be owned by the compute user");
+ }
+ if metadata.permissions().mode() & 0o777 != 0o600 {
+ bail!("{description} must have mode 0600");
+ }
+ }
+ Ok(())
+}
+
+#[cfg(unix)]
+fn effective_uid() -> u32 {
+ nix::unistd::geteuid().as_raw()
+}
+
+#[cfg(unix)]
+fn same_file(left: &Metadata, right: &Metadata) -> bool {
+ left.dev() == right.dev() && left.ino() == right.ino()
+}
+
+#[cfg(not(unix))]
+fn same_file(_left: &Metadata, _right: &Metadata) -> bool {
+ true
+}
+
+fn lock_file_name(file_name: &OsStr) -> OsString {
+ let mut name = OsString::from(".");
+ name.push(file_name);
+ name.push(".lock");
+ name
+}
+
+fn temp_file_name(file_name: &OsStr) -> OsString {
+ let mut name = OsString::from(".");
+ name.push(file_name);
+ name.push(".");
+ name.push(Uuid::new_v4().simple().to_string());
+ name.push(".tmp");
+ name
+}
+
+fn sync_directory(parent: &Path) -> Result<()> {
+ File::open(parent)
+ .and_then(|directory| directory.sync_all())
+ .context("sync compute ownership state directory")
+}
+
+fn spec_hash(spec: &ComputeSpec) -> Result<String> {
+ let value = serde_json::to_value(spec).context("serialize owned compute spec")?;
+ let mut canonical = Vec::new();
+ write_canonical_json(&value, &mut canonical)?;
+ Ok(hex::encode(Sha256::digest(canonical)))
+}
+
+fn write_canonical_json(value: &Value, output: &mut Vec<u8>) -> Result<()> {
+ match value {
+ Value::Null => output.extend_from_slice(b"null"),
+ Value::Bool(value) => output.extend_from_slice(if *value { b"true" } else { b"false" }),
+ Value::Number(value) => output.extend_from_slice(value.to_string().as_bytes()),
+ Value::String(value) => serde_json::to_writer(output, value)?,
+ Value::Array(values) => {
+ output.push(b'[');
+ for (index, value) in values.iter().enumerate() {
+ if index > 0 {
+ output.push(b',');
+ }
+ write_canonical_json(value, output)?;
+ }
+ output.push(b']');
+ }
+ Value::Object(values) => {
+ output.push(b'{');
+ let mut keys: Vec<_> = values.keys().collect();
+ keys.sort_unstable();
+ for (index, key) in keys.into_iter().enumerate() {
+ if index > 0 {
+ output.push(b',');
+ }
+ serde_json::to_writer(&mut *output, key)?;
+ output.push(b':');
+ write_canonical_json(&values[key], output)?;
+ }
+ output.push(b'}');
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[cfg(unix)]
+ use std::process::{Command, Stdio};
+
+ struct TestPath {
+ directory: PathBuf,
+ state: PathBuf,
+ }
+
+ impl TestPath {
+ fn new() -> Self {
+ let directory = std::env::temp_dir().join(format!(
+ "compute-ownership-directory-{}",
+ Uuid::new_v4().simple()
+ ));
+ fs::create_dir(&directory).unwrap();
+ #[cfg(unix)]
+ fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)).unwrap();
+ let state = directory.join("ownership.json");
+ Self { directory, state }
+ }
+ }
+
+ impl Drop for TestPath {
+ fn drop(&mut self) {
+ let _ = fs::remove_dir_all(&self.directory);
+ }
+ }
+
+ fn headers(token: &str) -> HeaderMap {
+ let mut headers = HeaderMap::new();
+ headers.insert(HAKOPOD_OWNERSHIP_TOKEN_HEADER, token.parse().unwrap());
+ headers
+ }
+
+ fn spec(token: &str) -> ComputeSpec {
+ let mut spec = ComputeSpec::default();
+ spec.operation_uuid = Some(token.to_owned());
+ spec
+ }
+
+ #[test]
+ fn configure_is_exactly_replayable_and_restart_safe() {
+ let paths = TestPath::new();
+ let token = Uuid::new_v4().simple().to_string();
+ let ownership = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ let request = spec(&token);
+ ownership
+ .authorize_configure(&headers(&token), &request)
+ .unwrap();
+ ownership
+ .authorize_configure(&headers(&token), &request)
+ .unwrap();
+
+ drop(ownership);
+ let restored = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ assert_eq!(restored.token().unwrap().as_deref(), Some(token.as_str()));
+ restored
+ .authorize_configure(&headers(&token), &request)
+ .unwrap();
+ }
+
+ #[test]
+ fn lifetime_lock_rejects_a_second_handle_until_release() {
+ let paths = TestPath::new();
+ let token = Uuid::new_v4().simple().to_string();
+ let first = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ first
+ .authorize_configure(&headers(&token), &spec(&token))
+ .unwrap();
+ assert!(ComputeOwnership::load(Some(paths.state.clone().into_os_string())).is_err());
+ drop(first);
+ let restored = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ restored
+ .authorize_spec_replay(&headers(&token), &spec(&token))
+ .unwrap();
+ }
+
+ #[test]
+ fn configure_rejects_foreign_token_and_changed_spec() {
+ let paths = TestPath::new();
+ let token = Uuid::new_v4().simple().to_string();
+ let foreign = Uuid::new_v4().simple().to_string();
+ let ownership = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ let request = spec(&token);
+ ownership
+ .authorize_configure(&headers(&token), &request)
+ .unwrap();
+ assert!(matches!(
+ ownership.authorize_configure(&headers(&foreign), &spec(&foreign)),
+ Err(OwnershipError::Conflict)
+ ));
+ let mut changed = spec(&token);
+ changed.project_id = Some("changed".to_owned());
+ assert!(matches!(
+ ownership.authorize_configure(&headers(&token), &changed),
+ Err(OwnershipError::Conflict)
+ ));
+ }
+
+ #[test]
+ fn terminate_requires_exact_token_and_persists_tombstone() {
+ let paths = TestPath::new();
+ let token = Uuid::new_v4().simple().to_string();
+ let foreign = Uuid::new_v4().simple().to_string();
+ let ownership = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ ownership
+ .authorize_configure(&headers(&token), &spec(&token))
+ .unwrap();
+ assert!(matches!(
+ ownership.authorize_terminate(&headers(&foreign)),
+ Err(OwnershipError::Conflict)
+ ));
+ ownership.authorize_terminate(&headers(&token)).unwrap();
+ drop(ownership);
+ let restored = ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ assert!(matches!(
+ restored.authorize_configure(&headers(&token), &spec(&token)),
+ Err(OwnershipError::Conflict)
+ ));
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn load_rejects_public_parent_and_symlinked_state() {
+ let paths = TestPath::new();
+ fs::set_permissions(&paths.directory, fs::Permissions::from_mode(0o755)).unwrap();
+ assert!(ComputeOwnership::load(Some(paths.state.clone().into_os_string())).is_err());
+ fs::set_permissions(&paths.directory, fs::Permissions::from_mode(0o700)).unwrap();
+ let target = paths.directory.join("target.json");
+ fs::write(&target, b"{}").unwrap();
+ fs::set_permissions(&target, fs::Permissions::from_mode(0o600)).unwrap();
+ std::os::unix::fs::symlink(&target, &paths.state).unwrap();
+ assert!(ComputeOwnership::load(Some(paths.state.clone().into_os_string())).is_err());
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn lifetime_lock_subprocess_helper() {
+ let Some(state) = std::env::var_os("HAKOPOD_OWNERSHIP_TEST_CHILD_STATE") else {
+ return;
+ };
+ let ready = PathBuf::from(std::env::var_os("HAKOPOD_OWNERSHIP_TEST_CHILD_READY").unwrap());
+ let release =
+ PathBuf::from(std::env::var_os("HAKOPOD_OWNERSHIP_TEST_CHILD_RELEASE").unwrap());
+ let ownership = ComputeOwnership::load(Some(state)).unwrap();
+ fs::write(&ready, b"ready").unwrap();
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !release.exists() && Instant::now() < deadline {
+ thread::sleep(Duration::from_millis(10));
+ }
+ assert!(release.exists(), "parent did not release child lock holder");
+ drop(ownership);
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn lifetime_lock_excludes_a_concurrent_process() {
+ let paths = TestPath::new();
+ let ready = paths.directory.join("child-ready");
+ let release = paths.directory.join("child-release");
+ let mut child = Command::new(std::env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "ownership::tests::lifetime_lock_subprocess_helper",
+ "--nocapture",
+ ])
+ .env("HAKOPOD_OWNERSHIP_TEST_CHILD_STATE", &paths.state)
+ .env("HAKOPOD_OWNERSHIP_TEST_CHILD_READY", &ready)
+ .env("HAKOPOD_OWNERSHIP_TEST_CHILD_RELEASE", &release)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .unwrap();
+ let ready_deadline = Instant::now() + Duration::from_secs(5);
+ while !ready.exists() && Instant::now() < ready_deadline {
+ if let Some(status) = child.try_wait().unwrap() {
+ panic!("lock holder exited before readiness: {status}");
+ }
+ thread::sleep(Duration::from_millis(10));
+ }
+ assert!(ready.exists(), "lock holder did not become ready");
+ let error = match ComputeOwnership::load(Some(paths.state.clone().into_os_string())) {
+ Ok(_) => panic!("second process acquired live compute authority"),
+ Err(error) => error,
+ };
+ assert!(
+ error
+ .to_string()
+ .contains("timed out locking compute ownership state")
+ );
+ fs::write(&release, b"release").unwrap();
+ assert!(child.wait().unwrap().success());
+ ComputeOwnership::load(Some(paths.state.clone().into_os_string())).unwrap();
+ }
+}
diff --git a/compute_tools/src/spec_apply.rs b/compute_tools/src/spec_apply.rs
index 90c0e234d575e2cd9985570c6fa54e563418cfb8..172ee655c5a23224ca6258e9a14b64dd68ecbd9a 100644
--- a/compute_tools/src/spec_apply.rs
+++ b/compute_tools/src/spec_apply.rs
@@ -44,7 +44,11 @@ pub fn apply_spec_sql(
concurrency: usize,
) -> Result<()> {
info!("Applying config with max {} concurrency", concurrency);
- debug!("Config: {:?}", spec);
+ if self.params.ownership_state_path.is_some() {
+ debug!("Applying owned compute configuration; sensitive spec omitted");
+ } else {
+ debug!("Config: {:?}", spec);
+ }
let rt = tokio::runtime::Handle::current();
rt.block_on(async {
diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs
index 2b81c3957c8a1a2eeab84db137f36dc2a4f4c064..3f8097daf1d2adf3ea25c385f9519dcaf4146c83 100644
--- a/control_plane/src/bin/neon_local.rs
+++ b/control_plane/src/bin/neon_local.rs
@@ -1130,6 +1130,8 @@ async fn handle_tenant(subcmd: &TenantCmd, env: &mut local_env::LocalEnv) -> any
.tenant_timeline_create(
tenant_id,
TimelineCreateRequest {
+ ownership_token: None,
+ ownership_request_hash: None,
new_timeline_id,
mode: pageserver_api::models::TimelineCreateRequestMode::Bootstrap {
existing_initdb_timeline_id: None,
@@ -1194,6 +1196,8 @@ async fn handle_timeline(cmd: &TimelineCmd, env: &mut local_env::LocalEnv) -> Re
let storage_controller = StorageController::from_env(env);
let create_req = TimelineCreateRequest {
+ ownership_token: None,
+ ownership_request_hash: None,
new_timeline_id,
mode: pageserver_api::models::TimelineCreateRequestMode::Bootstrap {
existing_initdb_timeline_id: None,
@@ -1259,6 +1263,8 @@ async fn handle_timeline(cmd: &TimelineCmd, env: &mut local_env::LocalEnv) -> Re
};
let pg_version = PgVersionId::from(args.pg_version);
let req = safekeeper_api::models::TimelineCreateRequest {
+ ownership_token: None,
+ ownership_request_hash: None,
tenant_id,
timeline_id,
mconf,
@@ -1290,6 +1296,8 @@ async fn handle_timeline(cmd: &TimelineCmd, env: &mut local_env::LocalEnv) -> Re
let start_lsn = args.ancestor_start_lsn;
let storage_controller = StorageController::from_env(env);
let create_req = TimelineCreateRequest {
+ ownership_token: None,
+ ownership_request_hash: None,
new_timeline_id,
mode: pageserver_api::models::TimelineCreateRequestMode::Branch {
ancestor_timeline_id,
diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs
index 635b1858ecce19e7f91a8d879267dd7e711f4a48..b46680d0453002fc6b5662dbdc3f6794121f89a0 100644
--- a/control_plane/storcon_cli/src/main.rs
+++ b/control_plane/storcon_cli/src/main.rs
@@ -511,7 +511,10 @@ async fn main() -> anyhow::Result<()> {
}
Command::TenantDelete { tenant_id } => {
let status = vps_client
- .tenant_delete(TenantShardId::unsharded(tenant_id))
+ .tenant_delete(
+ TenantShardId::unsharded(tenant_id),
+ &pageserver_api::models::TenantDeleteRequest::default(),
+ )
.await?;
tracing::info!("Delete status: {}", status);
}
diff --git a/hakopod-ownership/OWNERSHIP_PROTOCOL.md b/hakopod-ownership/OWNERSHIP_PROTOCOL.md
new file mode 100644
index 0000000000000000000000000000000000000000..0cf1820327bf52f580ab7c3cf716736532d991ca
--- /dev/null
+++ b/hakopod-ownership/OWNERSHIP_PROTOCOL.md
@@ -0,0 +1,86 @@
+# Hakopod provider ownership protocol
+
+Status: patch candidate. Strict ownership is active only when the storage
+controller, pageservers, safekeepers, compute control service, and proxy are
+started with their Hakopod ownership settings. The storage controller advertises
+`hakopod-ownership-v1` only in that mode.
+
+The caller generates a cryptographically random 128-bit UUID token and persists it
+before its first provider request. It sends the token in
+`Hakopod-Ownership-Token`. The provider canonicalizes the resource kind,
+external key, and all creation parameters and hashes them with SHA-256.
+
+The storage controller persists one immutable ledger row per resource key and
+one per token. A replay with the same token and parameter hash returns the
+stored result. A different token for the same key, or the same token with a
+different hash, returns 409. Describe responses return the token. Delete
+requires the token and records a tombstone before releasing the external key.
+
+The contract covers tenant creation, timeline creation, pageserver
+registration, and safekeeper registration. Timeline creation must propagate
+the token to pageservers and safekeepers; those services must enforce the same
+key/token rule before mutation. Legacy mutation routes must be disabled while
+strict ownership mode is enabled, otherwise an actor can bypass the ledger.
+
+Tenant deletion is a three-step protocol. `validate_only=true` holds the
+controller tenant-operation lock and performs a read-only, bounded preflight
+without loading a detached tenant or changing the ledger, placement, pending
+operations, tombstones, or provider data. It validates every child ledger row,
+the matching safekeeper records on the union of current and replacement sets,
+and every remote pageserver descendant.
+
+`prepare_delete=true` repeats the preflight, transitions the tenant ledger to
+`deleting` to block new child reservations, and asks the pageserver to publish
+an immutable tenant fence outside the deletable prefix. The pageserver binds
+that fence to the exact authorized child set, rescans after publishing it, and
+returns a provider-generated retry token. The controller stores that exact
+token durably before replying. A crash after fence publication resumes by
+reading the same fence token; a conflicting token or child set returns 409.
+Hakopod must complete preparation before changing claims, compute, timelines,
+or tenant data. The final delete supplies the ownership and preparation tokens
+in headers, revalidates all remaining descendants, and then creates child
+tombstones and deletes data. Tokens never appear in request URLs.
+
+Timeline reservation uses an immutable conditional-create fence outside the
+timeline data prefix. Before publishing that fence, and again afterward, the
+pageserver lists the exact timeline prefix without a delimiter and with a
+one-key limit. Any pre-existing key rejects adoption. Tenant and timeline
+deletion fences remain outside tenant data prefixes so deletion cannot erase
+them. Strict-mode timeline creation checks the tenant fence before and after
+reservation, and the controller refuses child ledger reservation after the
+parent tenant starts deletion.
+
+An exact retry may resume timeline data written after its immutable reservation
+even when no local timeline survived the interruption. The authenticated
+controller request, matching ledger token and request hash, and matching remote
+reservation together provide authority for that recovery; a fresh reservation
+still requires an empty remote prefix. During tenant deletion, controller
+placement remains durable until both pageserver and safekeeper children are
+absent, so a restart can continue cleanup before removing the final ledger row.
+
+Compute uses the existing `ComputeSpec.operation_uuid`. Hakopod supplies the
+same durable token there and accepts a configured compute only when `/status`
+returns that exact value together with the expected tenant and timeline.
+
+## Provider state machine
+
+`reserved -> applying -> completed -> deleting -> tombstoned`
+
+`failed` is terminal and retains the key. A worker may resume `reserved` or
+`applying` only with the same token and hash. Completion stores the exact
+provider response identity in `result` before replying.
+
+## Crash windows
+
+1. Before reserve commit: no provider mutation occurred; retry may reserve.
+2. After reserve commit, before mutation: replay resumes under the same token.
+3. During mutation: the resource service checks the same token, so replay can
+ only resume or observe that token's resource.
+4. After mutation, before ledger completion: reconciliation reads the token
+ from the resource service, verifies the parameter hash and identity, then
+ completes the ledger.
+5. After completion, before response: replay returns the stored result.
+6. During delete: the tombstone fences both stale create replay and key reuse.
+
+Configuration equality, deterministic identifiers, authenticated inspection,
+and a Hakopod-local pending intent are never sufficient proof of ownership.
diff --git a/hakopod-ownership/contract.rs b/hakopod-ownership/contract.rs
new file mode 100644
index 0000000000000000000000000000000000000000..e4f10a132e7bd9027f542a64bdfe05f120454b52
--- /dev/null
+++ b/hakopod-ownership/contract.rs
@@ -0,0 +1,46 @@
+//! Reference state machine for the provider-owned idempotency implemented by
+//! the storage controller and its downstream services.
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use uuid::Uuid;
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum State { Reserved, Applying, Completed, Failed, Deleting, Tombstoned }
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Record {
+ pub ownership_token: Uuid,
+ pub resource_kind: String,
+ pub external_key: String,
+ pub request_hash: [u8; 32],
+ pub state: State,
+ pub result: Option<Value>,
+ pub error_code: Option<String>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum Decision { Begin, Resume, Replay(Value), Conflict, Gone }
+
+impl Record {
+ pub fn decide(&self, token: Uuid, hash: [u8; 32]) -> Decision {
+ if self.ownership_token != token || self.request_hash != hash { return Decision::Conflict; }
+ match self.state {
+ State::Reserved | State::Applying => Decision::Resume,
+ State::Completed => Decision::Replay(self.result.clone().expect("completed result")),
+ State::Failed => Decision::Conflict,
+ State::Deleting | State::Tombstoned => Decision::Gone,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ fn record(state: State) -> Record { Record { ownership_token: Uuid::nil(), resource_kind: "tenant".into(), external_key: "tenant-1".into(), request_hash: [7; 32], state, result: Some(Value::String("identity".into())), error_code: None } }
+ #[test] fn exact_replay_returns_stored_identity() { assert!(matches!(record(State::Completed).decide(Uuid::nil(), [7; 32]), Decision::Replay(_))); }
+ #[test] fn foreign_token_is_rejected() { assert_eq!(record(State::Applying).decide(Uuid::from_u128(1), [7; 32]), Decision::Conflict); }
+ #[test] fn changed_parameters_are_rejected() { assert_eq!(record(State::Applying).decide(Uuid::nil(), [8; 32]), Decision::Conflict); }
+ #[test] fn deletion_fences_replay() { assert_eq!(record(State::Tombstoned).decide(Uuid::nil(), [7; 32]), Decision::Gone); }
+}
diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs
index a61d418dd18e9be86d66a57f6b684eefcd9e06fd..197814fbd645937d58a23234c625b41d331b99b1 100644
--- a/libs/compute_api/src/responses.rs
+++ b/libs/compute_api/src/responses.rs
@@ -144,12 +144,38 @@ pub struct ComputeStatusResponse {
pub start_time: DateTime<Utc>,
pub tenant: Option<String>,
pub timeline: Option<String>,
+ /// Durable control-plane operation that supplied the active spec. Callers
+ /// use this to distinguish their configure from an identical foreign one.
+ pub operation_uuid: Option<String>,
pub status: ComputeStatus,
#[serde(serialize_with = "rfc3339_serialize")]
pub last_active: Option<DateTime<Utc>>,
pub error: Option<String>,
}
+#[cfg(test)]
+mod compute_status_ownership_tests {
+ use super::*;
+
+ #[test]
+ fn status_round_trip_preserves_operation_uuid() {
+ let value = serde_json::json!({
+ "start_time": "2026-10-01T00:00:00Z",
+ "tenant": "11111111111111111111111111111111",
+ "timeline": "22222222222222222222222222222222",
+ "operation_uuid": "018f0000-0000-7000-8000-000000000001",
+ "status": "running",
+ "last_active": null,
+ "error": null
+ });
+ let parsed: ComputeStatusResponse = serde_json::from_value(value).unwrap();
+ assert_eq!(
+ parsed.operation_uuid.as_deref(),
+ Some("018f0000-0000-7000-8000-000000000001")
+ );
+ }
+}
+
#[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "snake_case")]
pub enum TerminateMode {
diff --git a/libs/pageserver_api/src/config.rs b/libs/pageserver_api/src/config.rs
index c6fd3757c604be8e57271e6f47f610a845cc0487..e88eca0836b7608be45d2fca6ca7baf08a1e7cca 100644
--- a/libs/pageserver_api/src/config.rs
+++ b/libs/pageserver_api/src/config.rs
@@ -267,6 +267,9 @@ pub struct ConfigToml {
pub tracing: Option<Tracing>,
pub enable_tls_page_service_api: bool,
pub dev_mode: bool,
+ /// Require the Hakopod ownership token and exact request hash on timeline
+ /// create and delete. Kept off by default for upstream and neon_local.
+ pub hakopod_ownership_v1: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub posthog_config: Option<PostHogConfig>,
pub timeline_import_config: TimelineImportConfig,
@@ -828,6 +831,7 @@ fn default() -> Self {
tracing: None,
enable_tls_page_service_api: false,
dev_mode: false,
+ hakopod_ownership_v1: false,
timeline_import_config: TimelineImportConfig {
import_job_concurrency: NonZeroUsize::new(32).unwrap(),
import_job_soft_size_limit: NonZeroUsize::new(256 * 1024 * 1024).unwrap(),
diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs
index 230c1f46ea68226aae6daba2663a1bff474f5d66..bf7c893d43a907621091edb4597164428c5c927b 100644
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -313,10 +313,43 @@ pub struct CompactInfoResponse {
#[derive(Serialize, Deserialize, Clone)]
pub struct TimelineCreateRequest {
pub new_timeline_id: TimelineId,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_token: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_request_hash: Option<String>,
#[serde(flatten)]
pub mode: TimelineCreateRequestMode,
}
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct TenantDeleteRequest {
+ #[serde(default)]
+ pub owned_timelines: Vec<OwnedTimelineDelete>,
+ #[serde(default)]
+ pub validate_only: bool,
+ #[serde(default)]
+ pub prepare_delete: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub expected_preflight_digest: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub delete_token: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OwnedTimelineDelete {
+ pub timeline_id: TimelineId,
+ pub ownership_token: String,
+ pub ownership_request_hash: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TenantDeletePreflightResponse {
+ pub schema_version: u8,
+ pub digest: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub delete_token: Option<String>,
+}
+
impl TimelineCreateRequest {
pub fn mode_tag(&self) -> &'static str {
match &self.mode {
@@ -1627,6 +1660,10 @@ pub struct TimelineInfo {
// HADRON: the largest LSN below which all page updates have been included in the image layers.
#[serde(skip_serializing_if = "Option::is_none")]
pub image_consistent_lsn: Option<Lsn>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ownership_token: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ownership_request_hash: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml
index bc70904958d685c27eaa2b87336870504781ebd3..651c50304cfa6d523857e767d6843aeebcc2f752 100644
--- a/libs/remote_storage/Cargo.toml
+++ b/libs/remote_storage/Cargo.toml
@@ -28,6 +28,7 @@ tokio-stream.workspace = true
tokio-util = { workspace = true, features = ["compat"] }
toml_edit.workspace = true
tracing.workspace = true
+thiserror.workspace = true
scopeguard.workspace = true
metrics.workspace = true
utils = { path = "../utils", default-features = false }
diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs
index db30829216b5c2dc07aed98aafd58f8c83f6d860..d4985ad402a538298c5375027b0e5a7a88092ad0 100644
--- a/libs/remote_storage/src/azure_blob.rs
+++ b/libs/remote_storage/src/azure_blob.rs
@@ -65,6 +65,40 @@ pub struct AzureBlobStorage {
}
impl AzureBlobStorage {
+ pub(crate) async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ let _permit = self
+ .permit(RequestKind::Put, cancel)
+ .await
+ .map_err(|error| crate::ConditionalCreateError::Other(error.into()))?;
+ let blob = self.client.blob_client(self.relative_path_to_name(to));
+ let builder = blob
+ .put_block_blob(azure_core::Body::from(data))
+ .if_match(IfMatchCondition::NotMatch("*".to_string()));
+ let result = tokio::select! {
+ result = tokio::time::timeout(self.timeout, builder.into_future()) => result,
+ _ = cancel.cancelled() => return Err(crate::ConditionalCreateError::Other(TimeoutOrCancel::Cancel.into())),
+ };
+ match result {
+ Ok(Ok(_)) => Ok(()),
+ Ok(Err(error))
+ if error
+ .as_http_error()
+ .is_some_and(|error| error.status() == StatusCode::PreconditionFailed) =>
+ {
+ Err(crate::ConditionalCreateError::AlreadyExists)
+ }
+ Ok(Err(error)) => Err(crate::ConditionalCreateError::Other(error.into())),
+ Err(_) => Err(crate::ConditionalCreateError::Other(
+ TimeoutOrCancel::Timeout.into(),
+ )),
+ }
+ }
+
pub fn new(
azure_config: &AzureConfig,
timeout: Duration,
@@ -517,6 +551,16 @@ fn to_download_error(error: azure_core::Error) -> DownloadError {
}
impl RemoteStorage for AzureBlobStorage {
+ /// Atomically publishes a bounded ownership marker without replacing it.
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ AzureBlobStorage::upload_if_absent(self, data, to, cancel).await
+ }
+
fn list_streaming(
&self,
prefix: Option<&RemotePath>,
diff --git a/libs/remote_storage/src/gcs_bucket.rs b/libs/remote_storage/src/gcs_bucket.rs
index b21825ddb35c6eb24fbc54227b88125ccf7048f6..737d59bb9bdf5fa7352271d89b96483170f3c058 100644
--- a/libs/remote_storage/src/gcs_bucket.rs
+++ b/libs/remote_storage/src/gcs_bucket.rs
@@ -3,10 +3,10 @@
pub(super) use crate::metrics::RequestKind;
use crate::metrics::{AttemptOutcome, start_counting_cancelled_wait, start_measuring_requests};
use crate::{
- ConcurrencyLimiter, Download, DownloadError, DownloadOpts, GCS_SCOPES, Listing, ListingMode,
- ListingObject, MAX_KEYS_PER_DELETE_GCS, REMOTE_STORAGE_PREFIX_SEPARATOR, RemotePath,
- RemoteStorage, StorageMetadata, TimeTravelError, TimeoutOrCancel, GCSVersion, VersionId,
- GCSVersionListing,
+ ConcurrencyLimiter, Download, DownloadError, DownloadOpts, GCS_SCOPES, GCSVersion,
+ GCSVersionListing, Listing, ListingMode, ListingObject, MAX_KEYS_PER_DELETE_GCS,
+ REMOTE_STORAGE_PREFIX_SEPARATOR, RemotePath, RemoteStorage, StorageMetadata, TimeTravelError,
+ TimeoutOrCancel, VersionId,
};
use anyhow::Context;
use azure_core::Etag;
@@ -54,6 +54,11 @@ pub struct GCSBucket {
pub timeout: Duration,
}
+enum GcsPutOutcome {
+ Uploaded,
+ AlreadyExists,
+}
+
struct GetObjectRequest {
bucket: String,
key: String,
@@ -64,6 +69,35 @@ struct GetObjectRequest {
// ---------
impl GCSBucket {
+ pub(crate) async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ let _permit = self
+ .permit(RequestKind::Put, cancel)
+ .await
+ .map_err(|error| crate::ConditionalCreateError::Other(error.into()))?;
+ let size = data.len();
+ let stream = futures::stream::once(async move { Ok::<Bytes, std::io::Error>(data) });
+ let upload = self.put_object(stream, size, to, cancel, None, true);
+ let result = tokio::select! {
+ result = tokio::time::timeout(self.timeout, upload) => result,
+ _ = cancel.cancelled() => return Err(crate::ConditionalCreateError::Other(TimeoutOrCancel::Cancel.into())),
+ };
+ match result {
+ Ok(Ok(GcsPutOutcome::Uploaded)) => Ok(()),
+ Ok(Ok(GcsPutOutcome::AlreadyExists)) => {
+ Err(crate::ConditionalCreateError::AlreadyExists)
+ }
+ Ok(Err(error)) => Err(crate::ConditionalCreateError::Other(error)),
+ Err(_) => Err(crate::ConditionalCreateError::Other(
+ TimeoutOrCancel::Timeout.into(),
+ )),
+ }
+ }
+
pub async fn new(remote_storage_config: &GCSConfig, timeout: Duration) -> anyhow::Result<Self> {
tracing::debug!(
"creating remote storage for gcs bucket {}",
@@ -188,16 +222,15 @@ async fn list_versions_with_permit(
max_keys: Option<NonZeroU32>,
cancel: &CancellationToken,
) -> Result<crate::GCSVersionListing, DownloadError> {
-
let warn_threshold = 3;
let max_retries = 10;
let is_permanent = |e: &_| matches!(e, DownloadError::Cancelled);
-
+
// GCS only has versions, which may contain 'deleted_at'.
let mut versions = crate::GCSVersionListing::default();
let mut continuation_token = None;
let mut uri: String;
-
+
let list_prefix = prefix
.map(|p| self.relative_path_to_gcs_object(p))
.or_else(|| {
@@ -207,7 +240,7 @@ async fn list_versions_with_permit(
})
})
.unwrap();
-
+
let mut versions_base_uri = format!(
"https://storage.googleapis.com/storage/v1/b/{}/o?prefix={}&versions=true",
self.bucket_name.clone(),
@@ -220,23 +253,21 @@ async fn list_versions_with_permit(
REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()
));
}
-
+
loop {
-
match &continuation_token {
Some(token) => {
uri = format!("{}&pageToken={}", &versions_base_uri, token);
- },
+ }
None => {
uri = versions_base_uri.clone();
- },
+ }
}
-
+
let mut req_uri = versions_base_uri.clone();
let response = backoff::retry(
|| async {
-
// fetch an array of results, keep looping to get them
let op = Client::new()
.get(&uri)
@@ -245,74 +276,76 @@ async fn list_versions_with_permit(
.token(GCS_SCOPES)
.await
.map_err(|e: gcp_auth::Error| DownloadError::Other(e.into()))?
- .as_str()
+ .as_str(),
)
.send();
-
- tokio::select! {
- res = op => res.map_err(|e| DownloadError::Other(e.into())),
- _ = cancel.cancelled() => Err(DownloadError::Cancelled),
- }
- },
- is_permanent,
- warn_threshold,
- max_retries,
- "listing object versions",
- cancel,
- )
- .await
- .ok_or_else(|| DownloadError::Cancelled)
- .and_then(|x| x)?;
-
- let res = response.json::<GCSListResponse>()
+ tokio::select! {
+ res = op => res.map_err(|e| DownloadError::Other(e.into())),
+ _ = cancel.cancelled() => Err(DownloadError::Cancelled),
+ }
+ },
+ is_permanent,
+ warn_threshold,
+ max_retries,
+ "listing object versions",
+ cancel,
+ )
+ .await
+ .ok_or_else(|| DownloadError::Cancelled)
+ .and_then(|x| x)?;
+
+ let res = response
+ .json::<GCSListResponse>()
.await
.map_err(|e| DownloadError::Other(e.into()))?;
-
- // fill up our results vec,
+
+ // fill up our results vec,
continuation_token = res.next_page_token;
-
- let version_listing =
- res.items
- .ok_or_else(|| DownloadError::Other(anyhow::anyhow!("no items returned")))?
- .into_iter()
- .map(| GCSObject { name, updated, time_deleted, generation, .. } | {
+
+ let version_listing = res
+ .items
+ .ok_or_else(|| DownloadError::Other(anyhow::anyhow!("no items returned")))?
+ .into_iter()
+ .map(
+ |GCSObject {
+ name,
+ updated,
+ time_deleted,
+ generation,
+ ..
+ }| {
// don't `filter_map`, a `None` for `last_modified` ('updated') is bad for
// time travel, so catch it.
if updated.is_none() {
- return Err(
- DownloadError::Other(
- anyhow::anyhow!("no 'updated' field")
- )
- )
+ return Err(DownloadError::Other(anyhow::anyhow!(
+ "no 'updated' field"
+ )));
}
- Ok(
- GCSVersion {
- key: self.gcs_object_to_relative_path(&name),
- last_modified: to_system_time(updated).unwrap(),
- id: VersionId(generation.expect("no version id")),
- time_deleted: to_system_time(time_deleted),
- }
- )
- }).collect::<Result<Vec<GCSVersion>, _>>();
-
+ Ok(GCSVersion {
+ key: self.gcs_object_to_relative_path(&name),
+ last_modified: to_system_time(updated).unwrap(),
+ id: VersionId(generation.expect("no version id")),
+ time_deleted: to_system_time(time_deleted),
+ })
+ },
+ )
+ .collect::<Result<Vec<GCSVersion>, _>>();
+
versions.versions.extend(version_listing?);
if let Some(max_keys) = max_keys {
if versions.versions.len() >= max_keys.get().try_into().unwrap() {
- return Err(DownloadError::Other(
- anyhow::anyhow!("max keys reached")
- ));
+ return Err(DownloadError::Other(anyhow::anyhow!("max keys reached")));
}
}
-
+
if continuation_token.is_none() {
- break
+ break;
}
}
-
+
Ok(versions)
-
}
async fn put_object(
@@ -322,20 +355,26 @@ async fn put_object(
to: &RemotePath,
cancel: &CancellationToken,
metadata: Option<StorageMetadata>,
- ) -> anyhow::Result<()> {
-
+ create_only: bool,
+ ) -> anyhow::Result<GcsPutOutcome> {
// we removed the semaphore permit here which was duplicative of the outer upload() trait
// impl call. put_object only called by upload(), so safe to do. upload handles permit and
// timeout, we were getting deadlock with upload concurrency at 32, and 32 upload permits
// then trying to get a permit inside their put_object calls, timing out, retrying,
// ad-infinitum.
-
+
let kind = RequestKind::Put;
let started_at = start_measuring_requests(kind);
+ let precondition = if create_only {
+ "&ifGenerationMatch=0"
+ } else {
+ ""
+ };
let multipart_uri = format!(
- "https://storage.googleapis.com/upload/storage/v1/b/{}/o?uploadType=multipart",
- self.bucket_name.clone()
+ "https://storage.googleapis.com/upload/storage/v1/b/{}/o?uploadType=multipart{}",
+ self.bucket_name.clone(),
+ precondition
);
let mut metadata = metadata.clone();
@@ -358,7 +397,7 @@ async fn put_object(
let data_part = reqwest::multipart::Part::stream_with_length(stream_body, fs_size as u64)
.mime_str("application/octet-stream")?;
- let mut form = reqwest::multipart::Form::new()
+ let form = reqwest::multipart::Form::new()
.part("metadata", metadata_part)
.part("bodystream", data_part);
@@ -390,11 +429,14 @@ async fn put_object(
crate::metrics::BUCKET_METRICS
.req_seconds
.observe_elapsed(kind, &res, started_at);
-
+
match res {
Ok(res) => {
if !res.status().is_success() {
match res.status() {
+ reqwest::StatusCode::PRECONDITION_FAILED if create_only => {
+ Ok(GcsPutOutcome::AlreadyExists)
+ }
_ => Err(anyhow::anyhow!("GCS PUT error \n\t {:?}", res)),
}
} else {
@@ -413,7 +455,7 @@ async fn put_object(
));
};
- Ok(())
+ Ok(GcsPutOutcome::Uploaded)
}
}
Err(reqw) => Err(reqw.into()),
@@ -757,15 +799,15 @@ async fn get_object(
// 2. Byte Stream request
let mut headers = header::HeaderMap::new();
let bytes_range = match &request.range {
- Some(s) => header::HeaderValue::from_str(s).unwrap(),
- None => header::HeaderValue::from_static("bytes=0-"),
+ Some(s) => header::HeaderValue::from_str(s).unwrap(),
+ None => header::HeaderValue::from_static("bytes=0-"),
};
-
+
tracing::info!(
"performing object download with {:?} range header",
bytes_range
);
-
+
headers.insert(header::RANGE, bytes_range);
let encoded_path: String =
@@ -855,56 +897,59 @@ async fn get_object(
metadata,
})
}
-
+
async fn copy_object(
- &self,
+ &self,
from: &RemotePath,
to: &RemotePath,
cancel: &CancellationToken,
generation: Option<&String>,
) -> anyhow::Result<reqwest::RequestBuilder> {
+ let copy_from_path: String = url::form_urlencoded::byte_serialize(
+ self.relative_path_to_gcs_object(to)
+ .trim_start_matches("/")
+ .as_bytes(),
+ )
+ .collect();
- let copy_from_path: String =
- url::form_urlencoded::byte_serialize(
- self.relative_path_to_gcs_object(to)
- .trim_start_matches("/")
- .as_bytes()
- )
- .collect();
-
- let copy_to_path: String =
- url::form_urlencoded::byte_serialize(
- self.relative_path_to_gcs_object(to)
- .trim_start_matches("/")
- .as_bytes()
- )
- .collect();
-
- let mut copy_uri = format!(
- "https://storage.googleapis.com/storage/v1/b/{}/o/{}/rewriteTo/b/{}/o/{}",
- self.bucket_name.clone(),
- copy_from_path,
- self.bucket_name.clone(),
- copy_to_path,
- );
-
- if let Some(gen_id) = generation {
- copy_uri += gen_id;
- }
-
- Ok(
- Client::new()
- .post(copy_uri)
- .bearer_auth(self.token_provider.token(GCS_SCOPES).await?.as_str())
- .header(header::CONTENT_TYPE, "application/json")
- .header(header::CONTENT_LENGTH, "0")
- )
- }
+ let copy_to_path: String = url::form_urlencoded::byte_serialize(
+ self.relative_path_to_gcs_object(to)
+ .trim_start_matches("/")
+ .as_bytes(),
+ )
+ .collect();
-
+ let mut copy_uri = format!(
+ "https://storage.googleapis.com/storage/v1/b/{}/o/{}/rewriteTo/b/{}/o/{}",
+ self.bucket_name.clone(),
+ copy_from_path,
+ self.bucket_name.clone(),
+ copy_to_path,
+ );
+
+ if let Some(gen_id) = generation {
+ copy_uri += gen_id;
+ }
+
+ Ok(Client::new()
+ .post(copy_uri)
+ .bearer_auth(self.token_provider.token(GCS_SCOPES).await?.as_str())
+ .header(header::CONTENT_TYPE, "application/json")
+ .header(header::CONTENT_LENGTH, "0"))
+ }
}
impl RemoteStorage for GCSBucket {
+ /// Atomically publishes a bounded ownership marker without replacing it.
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ GCSBucket::upload_if_absent(self, data, to, cancel).await
+ }
+
// ---------------------------------------
// Neon wrappers for GCS client functions
// ---------------------------------------
@@ -1009,7 +1054,7 @@ fn list_streaming(
let mut result = Listing::default();
for res in keys.iter() {
-
+
let last_modified: SystemTime = to_system_time(res.updated.clone()).unwrap_or(SystemTime::now());
let size = res.size.unwrap_or(0) as u64;
@@ -1056,7 +1101,7 @@ fn list_streaming(
}
}
}
-
+
async fn copy(
&self,
from: &RemotePath,
@@ -1070,13 +1115,8 @@ async fn copy(
let timeout = tokio::time::sleep(self.timeout);
let started_at = start_measuring_requests(kind);
-
- let op = self.copy_object(
- from,
- to,
- cancel,
- None
- ).await?.send();
+
+ let op = self.copy_object(from, to, cancel, None).await?.send();
let res = tokio::select! {
res = op => res,
@@ -1094,8 +1134,6 @@ async fn copy(
Ok(())
}
-
-
async fn upload(
&self,
from: impl Stream<Item = std::io::Result<Bytes>> + Send + Sync + 'static,
@@ -1109,7 +1147,7 @@ async fn upload(
let started_at = start_measuring_requests(kind);
- let upload = self.put_object(from, from_size_bytes, to, cancel, metadata);
+ let upload = self.put_object(from, from_size_bytes, to, cancel, metadata, false);
let upload = tokio::time::timeout(self.timeout, upload);
@@ -1127,10 +1165,11 @@ async fn upload(
}
match res {
- Ok(Ok(_put)) => Ok(()),
- Ok(Err(sdk)) => {
- Err(sdk.into())
- }
+ Ok(Ok(GcsPutOutcome::Uploaded)) => Ok(()),
+ Ok(Ok(GcsPutOutcome::AlreadyExists)) => Err(anyhow::anyhow!(
+ "unconditional GCS upload returned AlreadyExists"
+ )),
+ Ok(Err(sdk)) => Err(sdk.into()),
Err(_timeout) => Err(TimeoutOrCancel::Timeout.into()),
}
}
@@ -1187,118 +1226,110 @@ async fn delete(&self, path: &RemotePath, cancel: &CancellationToken) -> anyhow:
}
async fn time_travel_recover(
- &self,
- prefix: Option<&RemotePath>,
- timestamp: SystemTime,
- done_if_after: SystemTime,
- cancel: &CancellationToken,
- complexity_limit: Option<NonZeroU32>,
+ &self,
+ prefix: Option<&RemotePath>,
+ timestamp: SystemTime,
+ done_if_after: SystemTime,
+ cancel: &CancellationToken,
+ complexity_limit: Option<NonZeroU32>,
) -> Result<(), TimeTravelError> {
-
- let kind = RequestKind::TimeTravel;
- let permit = self.permit(kind, cancel).await?;
-
- tracing::trace!("Target time: {timestamp:?}, done_if_after {done_if_after:?}");
-
- let mode = ListingMode::NoDelimiter;
- let version_listing = self
- .list_versions_with_permit(&permit, prefix, mode, complexity_limit, cancel)
- .await
- .map_err(|err| match err {
- DownloadError::Other(e) => TimeTravelError::Other(e),
- DownloadError::Cancelled => TimeTravelError::Cancelled,
- other => TimeTravelError::Other(other.into()),
- })?;
- let versions_and_deletes = version_listing.versions;
-
- tracing::info!(
- "Built list for time travel with {} versions and deletions",
- versions_and_deletes.len()
- );
-
- // Work on the list of references instead of the objects directly,
- // otherwise we get lifetime errors in the sort_by_key call below.
- let mut versions_and_deletes = versions_and_deletes.iter().collect::<Vec<_>>();
-
- versions_and_deletes.sort_by_key(|vd| (&vd.key, &vd.last_modified));
-
- let mut vds_for_key = HashMap::<_, Vec<_>>::new();
-
- for vd in &versions_and_deletes {
- let GCSVersion { key, .. } = &vd;
- if Some(vd.id.0.as_str()) == Some("null") {
- // TODO: check the behavior of using the SDK on a non-versioned container
- return Err(TimeTravelError::Other(anyhow::anyhow!(
- "Received ListVersions response for key={key} with version_id='null', \
+ let kind = RequestKind::TimeTravel;
+ let permit = self.permit(kind, cancel).await?;
+
+ tracing::trace!("Target time: {timestamp:?}, done_if_after {done_if_after:?}");
+
+ let mode = ListingMode::NoDelimiter;
+ let version_listing = self
+ .list_versions_with_permit(&permit, prefix, mode, complexity_limit, cancel)
+ .await
+ .map_err(|err| match err {
+ DownloadError::Other(e) => TimeTravelError::Other(e),
+ DownloadError::Cancelled => TimeTravelError::Cancelled,
+ other => TimeTravelError::Other(other.into()),
+ })?;
+ let versions_and_deletes = version_listing.versions;
+
+ tracing::info!(
+ "Built list for time travel with {} versions and deletions",
+ versions_and_deletes.len()
+ );
+
+ // Work on the list of references instead of the objects directly,
+ // otherwise we get lifetime errors in the sort_by_key call below.
+ let mut versions_and_deletes = versions_and_deletes.iter().collect::<Vec<_>>();
+
+ versions_and_deletes.sort_by_key(|vd| (&vd.key, &vd.last_modified));
+
+ let mut vds_for_key = HashMap::<_, Vec<_>>::new();
+
+ for vd in &versions_and_deletes {
+ let GCSVersion { key, .. } = &vd;
+ if Some(vd.id.0.as_str()) == Some("null") {
+ // TODO: check the behavior of using the SDK on a non-versioned container
+ return Err(TimeTravelError::Other(anyhow::anyhow!(
+ "Received ListVersions response for key={key} with version_id='null', \
indicating either disabled versioning, or legacy objects with null version id values"
- )));
- }
- tracing::trace!("Parsing version key={key} id={:?}", vd.id);
- vds_for_key.entry(key).or_default().push(vd);
- }
-
- let warn_threshold = 3;
- let max_retries = 10;
- let is_permanent = |e: &_| matches!(e, TimeTravelError::Cancelled);
-
- for (key, versions) in vds_for_key {
- let last_vd = versions.last().unwrap();
- let key = self.relative_path_to_gcs_object(key);
- if last_vd.last_modified > done_if_after {
- /// Case 1: we have a recent object outside of our restore window.
- tracing::trace!("Key {key} has version later than done_if_after, skipping");
- continue;
- }
- /// we get index in the array that we want whether its `v` or `e`
- let version_to_restore_to =
- match versions.binary_search_by_key(×tamp, |tpl| tpl.last_modified) {
- Ok(v) => v,
- Err(e) => e,
- };
-
- let mut do_delete = false;
- if version_to_restore_to == 0 {
- // All versions more recent, so the key didn't exist at the specified time point.
- tracing::trace!(
- "All {} versions more recent for {key}, deleting",
- versions.len()
- );
- do_delete = true;
-
- } else {
-
- let GCSVersion {
- id: VersionId(version_id),
- time_deleted: deletion_timestamp,
- ..
- } = &versions[version_to_restore_to - 1];
-
- // GCS only has 'timeDeleted', not a version object per delete + version.
- // A version is either replaced by an object or removed -- stomped or dropped.
- // If `timeDeleted` < `time_travel_timestamp`, obj was removed and ought to be deleted.
- // If its `None`, that means we have the most current object, no-op.
- // Else, it was the same as the `updated` / `timeCreated` of the subsequent version, and ought to be restored.
- match &deletion_timestamp {
-
- Some(time) => {
-
- if time < ×tamp {
- // Case 2: version was last marked deleted before `timestamp`
- do_delete = true;
-
- } else {
-
- // Case 3: restore state to this version via `copy_object`
- tracing::trace!("Copying old version {version_id} for {key}...");
-
- let source_id =
- format!("?sourceGeneration={version_id}");
-
- backoff::retry(
+ )));
+ }
+ tracing::trace!("Parsing version key={key} id={:?}", vd.id);
+ vds_for_key.entry(key).or_default().push(vd);
+ }
+
+ let warn_threshold = 3;
+ let max_retries = 10;
+ let is_permanent = |e: &_| matches!(e, TimeTravelError::Cancelled);
+
+ for (key, versions) in vds_for_key {
+ let last_vd = versions.last().unwrap();
+ let key = self.relative_path_to_gcs_object(key);
+ if last_vd.last_modified > done_if_after {
+ /// Case 1: we have a recent object outside of our restore window.
+ tracing::trace!("Key {key} has version later than done_if_after, skipping");
+ continue;
+ }
+ /// we get index in the array that we want whether its `v` or `e`
+ let version_to_restore_to =
+ match versions.binary_search_by_key(×tamp, |tpl| tpl.last_modified) {
+ Ok(v) => v,
+ Err(e) => e,
+ };
+
+ let mut do_delete = false;
+ if version_to_restore_to == 0 {
+ // All versions more recent, so the key didn't exist at the specified time point.
+ tracing::trace!(
+ "All {} versions more recent for {key}, deleting",
+ versions.len()
+ );
+ do_delete = true;
+ } else {
+ let GCSVersion {
+ id: VersionId(version_id),
+ time_deleted: deletion_timestamp,
+ ..
+ } = &versions[version_to_restore_to - 1];
+
+ // GCS only has 'timeDeleted', not a version object per delete + version.
+ // A version is either replaced by an object or removed -- stomped or dropped.
+ // If `timeDeleted` < `time_travel_timestamp`, obj was removed and ought to be deleted.
+ // If its `None`, that means we have the most current object, no-op.
+ // Else, it was the same as the `updated` / `timeCreated` of the subsequent version, and ought to be restored.
+ match &deletion_timestamp {
+ Some(time) => {
+ if time < ×tamp {
+ // Case 2: version was last marked deleted before `timestamp`
+ do_delete = true;
+ } else {
+ // Case 3: restore state to this version via `copy_object`
+ tracing::trace!("Copying old version {version_id} for {key}...");
+
+ let source_id = format!("?sourceGeneration={version_id}");
+
+ backoff::retry(
|| async {
-
+
let key_path = self.gcs_object_to_relative_path(&key);
-
+
let op = self.copy_object(
&key_path,
&key_path,
@@ -1306,7 +1337,7 @@ async fn time_travel_recover(
Some(&source_id),
).await.map_err(|e| TimeTravelError::Other(e.into()))?
.send();
-
+
tokio::select! {
res = op => res.map_err(|e| TimeTravelError::Other(e.into())),
_ = cancel.cancelled() => Err(TimeTravelError::Cancelled),
@@ -1321,29 +1352,29 @@ async fn time_travel_recover(
.await
.ok_or_else(|| TimeTravelError::Cancelled)
.and_then(|x| {x})?;
- tracing::info!(%version_id, %key, "Copied old version in GCS");
- }
- },
- _ => {
+ tracing::info!(%version_id, %key, "Copied old version in GCS");
+ }
+ }
+ _ => {
tracing::info!("most current object version, skipping");
- }
- }
- };
- if do_delete {
- tracing::trace!("Deleting {key}...");
- self.delete_oids(&[key], cancel, &permit)
- .await
- .map_err(|e| {
- // delete_oid0 will use TimeoutOrCancel
- if TimeoutOrCancel::caused_by_cancel(&e) {
- TimeTravelError::Cancelled
- } else {
- TimeTravelError::Other(e)
- }
- })?;
- }
- }
- Ok(())
+ }
+ }
+ };
+ if do_delete {
+ tracing::trace!("Deleting {key}...");
+ self.delete_oids(&[key], cancel, &permit)
+ .await
+ .map_err(|e| {
+ // delete_oid0 will use TimeoutOrCancel
+ if TimeoutOrCancel::caused_by_cancel(&e) {
+ TimeTravelError::Cancelled
+ } else {
+ TimeTravelError::Other(e)
+ }
+ })?;
+ }
+ }
+ Ok(())
}
async fn head_object(
&self,
@@ -1381,10 +1412,10 @@ async fn list_versions(
) -> Result<crate::VersionListing, DownloadError> {
let kind = RequestKind::ListVersions;
let permit = self.permit(kind, cancel).await?;
- Ok(
- self.list_versions_with_permit(&permit, prefix, mode, max_keys, cancel)
- .await?.into()
- )
+ Ok(self
+ .list_versions_with_permit(&permit, prefix, mode, max_keys, cancel)
+ .await?
+ .into())
}
}
diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs
index 14245522ed2acaf90d8299efb533603a801e18f2..08d7e6be5a9ef96856809d37310e4e7ce8bd9001 100644
--- a/libs/remote_storage/src/lib.rs
+++ b/libs/remote_storage/src/lib.rs
@@ -84,6 +84,15 @@
pub const MAX_KEYS_PER_DELETE_AZURE: usize = 256;
pub const MAX_KEYS_PER_DELETE_GCS: usize = 1000;
+const MAX_CONDITIONAL_CREATE_BYTES: usize = 64 << 10;
+
+#[derive(Debug, thiserror::Error)]
+pub enum ConditionalCreateError {
+ #[error("remote object already exists")]
+ AlreadyExists,
+ #[error(transparent)]
+ Other(#[from] anyhow::Error),
+}
const REMOTE_STORAGE_PREFIX_SEPARATOR: char = '/';
@@ -329,6 +338,14 @@ pub fn byte_range_header(&self) -> Option<String> {
/// providing basic CRUD operations for storage files.
#[allow(async_fn_in_trait)]
pub trait RemoteStorage: Send + Sync + 'static {
+ /// Atomically publishes a bounded ownership marker without replacing it.
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError>;
+
/// List objects in remote storage, with semantics matching AWS S3's [`ListObjectsV2`].
///
/// The stream is guaranteed to return at least one element, even in the case of errors
@@ -618,6 +635,27 @@ pub async fn upload(
}
}
+ /// Atomically creates an object and never replaces an existing value.
+ pub async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError> {
+ if data.len() > MAX_CONDITIONAL_CREATE_BYTES {
+ return Err(ConditionalCreateError::Other(anyhow::anyhow!(
+ "conditional-create marker exceeds {MAX_CONDITIONAL_CREATE_BYTES} bytes"
+ )));
+ }
+ match self {
+ Self::LocalFs(s) => s.upload_if_absent(data, to, cancel).await,
+ Self::AwsS3(s) => s.upload_if_absent(data, to, cancel).await,
+ Self::AzureBlob(s) => s.upload_if_absent(data, to, cancel).await,
+ Self::Unreliable(s) => s.upload_if_absent(data, to, cancel).await,
+ Self::GCS(s) => s.upload_if_absent(data, to, cancel).await,
+ }
+ }
+
/// See [`RemoteStorage::download`]
pub async fn download(
&self,
diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs
index 30690b1bdbfafb1e3bac38ea01b3dbee3d8ea187..f318cf3c924538328c483c562d090ae6af8092f6 100644
--- a/libs/remote_storage/src/local_fs.rs
+++ b/libs/remote_storage/src/local_fs.rs
@@ -21,8 +21,8 @@
use super::{RemoteStorage, StorageMetadata};
use crate::{
- Download, DownloadError, DownloadOpts, Etag, Listing, ListingMode, ListingObject,
- REMOTE_STORAGE_PREFIX_SEPARATOR, RemotePath, TimeTravelError, TimeoutOrCancel,
+ ConditionalCreateError, Download, DownloadError, DownloadOpts, Etag, Listing, ListingMode,
+ ListingObject, REMOTE_STORAGE_PREFIX_SEPARATOR, RemotePath, TimeTravelError, TimeoutOrCancel,
};
const LOCAL_FS_TEMP_FILE_SUFFIX: &str = "___temp";
@@ -34,6 +34,85 @@ pub struct LocalFs {
}
impl LocalFs {
+ pub(crate) async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError> {
+ match tokio::time::timeout(self.timeout, self.upload_if_absent_inner(data, to, cancel))
+ .await
+ {
+ Ok(result) => result,
+ Err(_) => Err(ConditionalCreateError::Other(
+ TimeoutOrCancel::Timeout.into(),
+ )),
+ }
+ }
+
+ async fn upload_if_absent_inner(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError> {
+ let target = to.with_base(&self.storage_root);
+ create_target_directory(&target)
+ .await
+ .map_err(ConditionalCreateError::Other)?;
+ if cancel.is_cancelled() {
+ return Err(ConditionalCreateError::Other(
+ TimeoutOrCancel::Cancel.into(),
+ ));
+ }
+ let temporary = path_with_suffix_extension(
+ &target,
+ &format!("conditional-{}", uuid::Uuid::new_v4().simple()),
+ );
+ let cleanup = scopeguard::guard(temporary.clone(), |path| {
+ let _ = std::fs::remove_file(path);
+ });
+ let mut file = fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(&temporary)
+ .await
+ .map_err(|error| ConditionalCreateError::Other(error.into()))?;
+ let write_result = tokio::select! {
+ _ = cancel.cancelled() => {
+ drop(file);
+ Err(ConditionalCreateError::Other(TimeoutOrCancel::Cancel.into()))
+ }
+ result = async {
+ file.write_all(&data).await?;
+ file.sync_all().await?;
+ Ok::<(), std::io::Error>(())
+ } => result.map_err(|error| ConditionalCreateError::Other(error.into()))
+ };
+ if let Err(error) = write_result {
+ return Err(error);
+ }
+ let publish = fs::hard_link(&temporary, &target).await;
+ drop(cleanup);
+ match publish {
+ Ok(()) => {
+ let parent = target.parent().ok_or_else(|| {
+ ConditionalCreateError::Other(anyhow::anyhow!(
+ "conditional create target has no parent"
+ ))
+ })?;
+ std::fs::File::open(parent)
+ .and_then(|directory| directory.sync_all())
+ .map_err(|error| ConditionalCreateError::Other(error.into()))?;
+ Ok(())
+ }
+ Err(error) if error.kind() == ErrorKind::AlreadyExists => {
+ Err(ConditionalCreateError::AlreadyExists)
+ }
+ Err(error) => Err(ConditionalCreateError::Other(error.into())),
+ }
+ }
+
/// Attempts to create local FS storage, along with its root directory.
/// Storage root will be created (if does not exist) and transformed into an absolute path (if passed as relative).
pub fn new(mut storage_root: Utf8PathBuf, timeout: Duration) -> anyhow::Result<Self> {
@@ -329,6 +408,16 @@ async fn upload0(
}
impl RemoteStorage for LocalFs {
+ /// Atomically publishes a bounded ownership marker without replacing it.
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError> {
+ LocalFs::upload_if_absent(self, data, to, cancel).await
+ }
+
fn list_streaming(
&self,
prefix: Option<&RemotePath>,
@@ -730,6 +819,57 @@ fn create_storage() -> anyhow::Result<(LocalFs, CancellationToken)> {
LocalFs::new(storage_root, Duration::from_secs(120)).map(|s| (s, CancellationToken::new()))
}
+ #[tokio::test]
+ async fn conditional_create_never_replaces_existing_object() -> anyhow::Result<()> {
+ let (storage, cancel) = create_storage()?;
+ let path = RemotePath::new(Utf8Path::new("ownership/tenant"))?;
+ storage
+ .upload_if_absent(Bytes::from_static(b"first"), &path, &cancel)
+ .await?;
+ let result = storage
+ .upload_if_absent(Bytes::from_static(b"second"), &path, &cancel)
+ .await;
+ assert!(matches!(
+ result,
+ Err(crate::ConditionalCreateError::AlreadyExists)
+ ));
+ let contents = read_and_check_metadata(&storage, &path, None).await?;
+ assert_eq!(contents, "first");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn concurrent_conditional_create_has_one_complete_winner() -> anyhow::Result<()> {
+ let (storage, cancel) = create_storage()?;
+ let path = RemotePath::new(Utf8Path::new("ownership/race"))?;
+ let left = storage.upload_if_absent(Bytes::from_static(b"left"), &path, &cancel);
+ let right = storage.upload_if_absent(Bytes::from_static(b"right"), &path, &cancel);
+ let (left, right) = tokio::join!(left, right);
+ assert_eq!(usize::from(left.is_ok()) + usize::from(right.is_ok()), 1);
+ assert!(
+ matches!(left, Err(crate::ConditionalCreateError::AlreadyExists))
+ || matches!(right, Err(crate::ConditionalCreateError::AlreadyExists))
+ );
+ let contents = read_and_check_metadata(&storage, &path, None).await?;
+ assert!(contents == "left" || contents == "right");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn cancelled_conditional_create_publishes_nothing() -> anyhow::Result<()> {
+ let (storage, cancel) = create_storage()?;
+ cancel.cancel();
+ let path = RemotePath::new(Utf8Path::new("ownership/cancelled"))?;
+ assert!(
+ storage
+ .upload_if_absent(Bytes::from_static(b"secret"), &path, &cancel)
+ .await
+ .is_err()
+ );
+ assert!(!path.with_base(&storage.storage_root).exists());
+ Ok(())
+ }
+
#[tokio::test]
async fn download_file() -> anyhow::Result<()> {
let (storage, cancel) = create_storage()?;
diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs
index 8a2e5bd10e4c1cebf0fa4046e74688c0c1d30c5c..1bb530270602b1d2560de6839b27c7a4fc6dfd19 100644
--- a/libs/remote_storage/src/s3_bucket.rs
+++ b/libs/remote_storage/src/s3_bucket.rs
@@ -68,6 +68,47 @@ struct GetObjectRequest {
version_id: Option<String>,
}
impl S3Bucket {
+ pub(crate) async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ let kind = RequestKind::Put;
+ let _permit = self
+ .permit(kind, cancel)
+ .await
+ .map_err(|error| crate::ConditionalCreateError::Other(error.into()))?;
+ let request = self
+ .client
+ .put_object()
+ .bucket(self.bucket_name.clone())
+ .key(self.relative_path_to_s3_object(to))
+ .set_storage_class(self.upload_storage_class.clone())
+ .content_length(data.len() as i64)
+ .if_none_match("*")
+ .body(ByteStream::from(data))
+ .send();
+ let result = tokio::select! {
+ result = tokio::time::timeout(self.timeout, request) => result,
+ _ = cancel.cancelled() => return Err(crate::ConditionalCreateError::Other(TimeoutOrCancel::Cancel.into())),
+ };
+ match result {
+ Ok(Ok(_)) => Ok(()),
+ Ok(Err(error))
+ if error
+ .raw_response()
+ .is_some_and(|response| response.status().as_u16() == 412) =>
+ {
+ Err(crate::ConditionalCreateError::AlreadyExists)
+ }
+ Ok(Err(error)) => Err(crate::ConditionalCreateError::Other(error.into())),
+ Err(_) => Err(crate::ConditionalCreateError::Other(
+ TimeoutOrCancel::Timeout.into(),
+ )),
+ }
+ }
+
/// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
pub async fn new(remote_storage_config: &S3Config, timeout: Duration) -> anyhow::Result<Self> {
tracing::debug!(
@@ -604,6 +645,16 @@ fn size_hint(&self) -> (usize, Option<usize>) {
}
impl RemoteStorage for S3Bucket {
+ /// Atomically publishes a bounded ownership marker without replacing it.
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), crate::ConditionalCreateError> {
+ S3Bucket::upload_if_absent(self, data, to, cancel).await
+ }
+
fn list_streaming(
&self,
prefix: Option<&RemotePath>,
diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs
index 58e491b56bccb00edc4f63186a6e4ada5150a9e3..be239afa53f534aa26ed96b2a0613878a2ae2d8c 100644
--- a/libs/remote_storage/src/simulate_failures.rs
+++ b/libs/remote_storage/src/simulate_failures.rs
@@ -15,8 +15,8 @@
use tokio_util::sync::CancellationToken;
use crate::{
- Download, DownloadError, DownloadOpts, GenericRemoteStorage, Listing, ListingMode, RemotePath,
- RemoteStorage, StorageMetadata, TimeTravelError,
+ ConditionalCreateError, Download, DownloadError, DownloadOpts, GenericRemoteStorage, Listing,
+ ListingMode, RemotePath, RemoteStorage, StorageMetadata, TimeTravelError,
};
pub struct UnreliableWrapper {
@@ -131,6 +131,17 @@ async fn delete_inner(
type VoidStorage = crate::LocalFs;
impl RemoteStorage for UnreliableWrapper {
+ async fn upload_if_absent(
+ &self,
+ data: Bytes,
+ to: &RemotePath,
+ cancel: &CancellationToken,
+ ) -> Result<(), ConditionalCreateError> {
+ self.attempt(RemoteOp::Upload(to.clone()))
+ .map_err(ConditionalCreateError::Other)?;
+ self.inner.upload_if_absent(data, to, cancel).await
+ }
+
fn list_streaming(
&self,
prefix: Option<&RemotePath>,
diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs
index b34ed947c01094f5e010d1ef7e413f4eb820fd77..1706bdba5522eacb986ec604898012ee88f8fb33 100644
--- a/libs/safekeeper_api/src/models.rs
+++ b/libs/safekeeper_api/src/models.rs
@@ -21,6 +21,10 @@ pub struct SafekeeperStatus {
#[derive(Serialize, Deserialize, Clone)]
pub struct TimelineCreateRequest {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_token: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_request_hash: Option<String>,
pub tenant_id: TenantId,
pub timeline_id: TimelineId,
pub mconf: Configuration,
@@ -295,9 +299,34 @@ pub struct SafekeeperUtilization {
pub timeline_count: u64,
}
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct OwnershipValidationEntry {
+ pub tenant_id: TenantId,
+ pub timeline_id: TimelineId,
+ pub ownership_token: String,
+ pub ownership_request_hash: String,
+ #[serde(default)]
+ pub allow_tombstoned: bool,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct OwnershipValidationRequest {
+ pub timelines: Vec<OwnershipValidationEntry>,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct OwnershipValidationResponse {
+ pub schema_version: u8,
+ pub digest: String,
+}
+
/// pull_timeline request body.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct PullTimelineRequest {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_token: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ownership_request_hash: Option<String>,
pub tenant_id: TenantId,
pub timeline_id: TimelineId,
pub http_hosts: Vec<String>,
diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml
index ceab5e1b446f7edcfb9f2534181cd07853c5bbdd..b7ca59bc0b5852d7759185b630dc32ad79da48df 100644
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -83,6 +83,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
serde_path_to_error.workspace = true
serde_with.workspace = true
serde.workspace = true
+sha2.workspace = true
smallvec.workspace = true
storage_broker.workspace = true
strum_macros.workspace = true
@@ -107,6 +108,7 @@ tracing.workspace = true
tracing-utils.workspace = true
url.workspace = true
utils.workspace = true
+uuid.workspace = true
wal_decoder.workspace = true
walkdir.workspace = true
workspace_hack.workspace = true
@@ -121,7 +123,6 @@ criterion.workspace = true
hex-literal.workspace = true
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
indoc.workspace = true
-uuid.workspace = true
rstest.workspace = true
[[bench]]
diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs
index 3867e536f476d0656fd7e22da20660579d95c7aa..e7b1a71e5d9fc01ccd0416e30f3ad99a01afb30f 100644
--- a/pageserver/client/src/mgmt_api.rs
+++ b/pageserver/client/src/mgmt_api.rs
@@ -221,10 +221,14 @@ pub async fn status(&self) -> Result<()> {
/// The tenant deletion API can return 202 if deletion is incomplete, or
/// 404 if it is complete. Callers are responsible for checking the status
/// code and retrying. Error codes other than 404 will return Err().
- pub async fn tenant_delete(&self, tenant_shard_id: TenantShardId) -> Result<StatusCode> {
+ pub async fn tenant_delete(
+ &self,
+ tenant_shard_id: TenantShardId,
+ request: &pageserver_api::models::TenantDeleteRequest,
+ ) -> Result<StatusCode> {
let uri = format!("{}/v1/tenant/{tenant_shard_id}", self.mgmt_api_endpoint);
- match self.request(Method::DELETE, &uri, ()).await {
+ match self.request(Method::DELETE, &uri, request).await {
Err(Error::ApiError(status_code, msg)) => {
if status_code == StatusCode::NOT_FOUND {
Ok(StatusCode::NOT_FOUND)
@@ -237,6 +241,16 @@ pub async fn tenant_delete(&self, tenant_shard_id: TenantShardId) -> Result<Stat
}
}
+ pub async fn tenant_delete_preflight(
+ &self,
+ tenant_shard_id: TenantShardId,
+ request: &pageserver_api::models::TenantDeleteRequest,
+ ) -> Result<pageserver_api::models::TenantDeletePreflightResponse> {
+ let uri = format!("{}/v1/tenant/{tenant_shard_id}", self.mgmt_api_endpoint);
+ let response = self.request(Method::DELETE, &uri, request).await?;
+ response.json().await.map_err(Error::ReceiveBody)
+ }
+
pub async fn tenant_time_travel_remote_storage(
&self,
tenant_shard_id: TenantShardId,
@@ -466,13 +480,19 @@ pub async fn timeline_delete(
&self,
tenant_shard_id: TenantShardId,
timeline_id: TimelineId,
+ ownership_token: Option<&str>,
) -> Result<StatusCode> {
let uri = format!(
"{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}",
self.mgmt_api_endpoint
);
- match self.request(Method::DELETE, &uri, ()).await {
+ let mut request = self.start_request(Method::DELETE, &uri).json(&());
+ if let Some(token) = ownership_token {
+ request = request.header("Hakopod-Ownership-Token", token);
+ }
+ let response = request.send().await.map_err(Error::ReceiveBody)?;
+ match response.error_from_body().await {
Err(Error::ApiError(status_code, msg)) => {
if status_code == StatusCode::NOT_FOUND {
Ok(StatusCode::NOT_FOUND)
diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs
index bb73ae1dd5140496b8ff721a43318be351b36760..bcb787cfc9750ec99410250e9b18895044cd1cc9 100644
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -260,6 +260,8 @@ pub struct PageServerConf {
/// Controls whether to collect all metrics on each scrape or to return potentially stale
/// results.
pub force_metric_collection_on_scrape: bool,
+
+ pub hakopod_ownership_v1: bool,
}
/// Token for authentication to safekeepers
@@ -437,6 +439,7 @@ pub fn parse_and_validate(
tracing,
enable_tls_page_service_api,
dev_mode,
+ hakopod_ownership_v1,
posthog_config,
timeline_import_config,
basebackup_cache_config,
@@ -502,6 +505,7 @@ pub fn parse_and_validate(
basebackup_cache_config,
image_layer_generation_large_timeline_threshold,
force_metric_collection_on_scrape,
+ hakopod_ownership_v1,
// ------------------------------------------------------------
// fields that require additional validation or custom handling
@@ -565,6 +569,11 @@ pub fn parse_and_validate(
// custom validation code that covers more than one field in isolation
// ------------------------------------------------------------
+ ensure!(
+ !(conf.dev_mode && conf.hakopod_ownership_v1),
+ "`hakopod_ownership_v1` cannot be enabled in development mode"
+ );
+
if [conf.http_auth_type, conf.pg_auth_type, conf.grpc_auth_type]
.contains(&AuthType::NeonJWT)
{
diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs
index 669eeffa321a34c96f965fe242c32350e2492b24..9c6434286a65363a3a7a51f8f27ab84148298c48 100644
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -34,7 +34,7 @@
DetachBehavior, DownloadRemoteLayersTaskSpawnRequest, IngestAuxFilesRequest,
ListAuxFilesRequest, LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease,
LsnLeaseRequest, OffloadedTimelineInfo, PageTraceEvent, ShardParameters, StatusResponse,
- TenantConfigPatchRequest, TenantConfigRequest, TenantDetails, TenantInfo,
+ TenantConfigPatchRequest, TenantConfigRequest, TenantDeleteRequest, TenantDetails, TenantInfo,
TenantLocationConfigRequest, TenantLocationConfigResponse, TenantScanRemoteStorageResponse,
TenantScanRemoteStorageShard, TenantShardLocation, TenantShardSplitRequest,
TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
@@ -368,6 +368,7 @@ fn from(value: crate::tenant::mgr::DeleteTenantError) -> Self {
use crate::tenant::mgr::DeleteTenantError::*;
match value {
SlotError(e) => e.into(),
+ OwnershipConflict(message) => ApiError::Conflict(message),
Other(o) => ApiError::InternalServerError(o),
Cancelled => ApiError::ShuttingDown,
}
@@ -486,6 +487,7 @@ async fn build_timeline_info_common(
let (rel_size_migration, rel_size_migrated_at) = timeline.get_rel_size_v2_status();
+ let (ownership_token, ownership_request_hash) = timeline.remote_client.ownership();
let info = TimelineInfo {
tenant_id: timeline.tenant_shard_id,
timeline_id: timeline.timeline_id,
@@ -524,6 +526,8 @@ async fn build_timeline_info_common(
walreceiver_status,
// HADRON
image_consistent_lsn: None,
+ ownership_token,
+ ownership_request_hash,
};
Ok(info)
}
@@ -595,6 +599,30 @@ async fn timeline_create_handler(
check_permission(&request, Some(tenant_shard_id.tenant_id))?;
let new_timeline_id = request_data.new_timeline_id;
+ let ownership_token = request_data.ownership_token.clone();
+ let ownership_request_hash = request_data.ownership_request_hash.clone();
+ if get_config(&request).hakopod_ownership_v1 {
+ let token = ownership_token
+ .as_deref()
+ .ok_or_else(|| ApiError::BadRequest(anyhow!("missing timeline ownership token")))?;
+ let supplied_hash = ownership_request_hash.as_deref().ok_or_else(|| {
+ ApiError::BadRequest(anyhow!("missing timeline ownership request hash"))
+ })?;
+ if token.len() != 32
+ || token == "00000000000000000000000000000000"
+ || !token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ || supplied_hash.len() != 64
+ || !supplied_hash
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ return Err(ApiError::BadRequest(anyhow!(
+ "invalid timeline ownership token or request hash"
+ )));
+ }
+ }
// fill in the default pg_version if not provided & convert request into domain model
let params: tenant::CreateTimelineParams = match request_data.mode {
TimelineCreateRequestMode::Bootstrap {
@@ -657,6 +685,15 @@ async fn timeline_create_handler(
tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
+ if let (Some(token), Some(request_hash)) = (
+ ownership_token.as_deref(),
+ ownership_request_hash.as_deref(),
+ ) {
+ tenant
+ .reserve_timeline_ownership(new_timeline_id, token, request_hash)
+ .await?;
+ }
+
// earlier versions of the code had pg_version and ancestor_lsn in the span
// => continue to provide that information, but, through a log message that doesn't require us to destructure
tracing::info!(?params, "creating timeline");
@@ -1157,6 +1194,25 @@ async fn timeline_delete_handler(
) -> Result<Response<Body>, ApiError> {
let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
+ let ownership_token = if get_config(&request).hakopod_ownership_v1 {
+ let token = request
+ .headers()
+ .get("Hakopod-Ownership-Token")
+ .and_then(|value| value.to_str().ok())
+ .filter(|token| {
+ token.len() == 32
+ && *token != "00000000000000000000000000000000"
+ && token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ })
+ .ok_or_else(|| {
+ ApiError::BadRequest(anyhow!("missing or invalid timeline ownership token"))
+ })?;
+ Some(token.to_string())
+ } else {
+ None
+ };
check_permission(&request, Some(tenant_shard_id.tenant_id))?;
let state = get_state(&request);
@@ -1177,6 +1233,11 @@ async fn timeline_delete_handler(
}
})?;
tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
+ if let Some(token) = ownership_token.as_deref() {
+ tenant
+ .tombstone_timeline_ownership(timeline_id, token)
+ .await?;
+ }
tenant.delete_timeline(timeline_id).instrument(info_span!("timeline_delete", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id))
.await?;
@@ -1292,25 +1353,35 @@ async fn tenant_status(
}
async fn tenant_delete_handler(
- request: Request<Body>,
+ mut request: Request<Body>,
_cancel: CancellationToken,
) -> Result<Response<Body>, ApiError> {
// TODO openapi spec
let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
check_permission(&request, Some(tenant_shard_id.tenant_id))?;
+ let verify_ownership = get_config(&request).hakopod_ownership_v1;
+ let delete_request = if verify_ownership {
+ json_request::<TenantDeleteRequest>(&mut request).await?
+ } else {
+ TenantDeleteRequest::default()
+ };
+
let state = get_state(&request);
- state
+ let result = state
.tenant_manager
- .delete_tenant(tenant_shard_id)
+ .delete_tenant(tenant_shard_id, verify_ownership.then_some(delete_request))
.instrument(info_span!("tenant_delete_handler",
tenant_id = %tenant_shard_id.tenant_id,
shard_id = %tenant_shard_id.shard_slug()
))
.await?;
- json_response(StatusCode::OK, ())
+ match result {
+ Some(preflight) => json_response(StatusCode::OK, preflight),
+ None => json_response(StatusCode::OK, ()),
+ }
}
/// HTTP endpoint to query the current tenant_size of a tenant.
diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs
index 91b717a2e948ed85f8c0acae7fafb5a27c15aa0e..66750f72df3fb0e46c553c98c78e91cd07d217fc 100644
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -17,6 +17,7 @@
use std::fmt::{Debug, Display};
use std::fs::File;
use std::future::Future;
+use std::num::NonZeroU32;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, Weak};
use std::time::{Duration, Instant, SystemTime};
@@ -24,11 +25,13 @@
use anyhow::{Context, bail};
use arc_swap::ArcSwap;
+use bytes::Bytes;
use camino::{Utf8Path, Utf8PathBuf};
use chrono::NaiveDateTime;
use enumset::EnumSet;
-use futures::StreamExt;
use futures::stream::FuturesUnordered;
+use futures::{StreamExt, TryStreamExt};
+use http_utils::error::ApiError;
use itertools::Itertools as _;
use once_cell::sync::Lazy;
pub use pageserver_api::models::TenantState;
@@ -39,7 +42,10 @@
};
use pageserver_api::shard::{ShardIdentity, ShardStripeSize, TenantShardId};
use postgres_ffi::PgMajorVersion;
-use remote_storage::{DownloadError, GenericRemoteStorage, TimeoutOrCancel};
+use remote_storage::{
+ ConditionalCreateError, DownloadError, DownloadOpts, GenericRemoteStorage, ListingMode,
+ TimeoutOrCancel,
+};
use remote_timeline_client::index::GcCompactionState;
use remote_timeline_client::manifest::{
LATEST_TENANT_MANIFEST_VERSION, OffloadedTimelineManifest, TenantManifest,
@@ -49,6 +55,7 @@
download_tenant_manifest,
};
use secondary::heatmap::{HeatMapTenant, HeatMapTimeline};
+use serde::{Deserialize, Serialize};
use storage_broker::BrokerClientChannel;
use timeline::compaction::{CompactionOutcome, GcCompactionQueue};
use timeline::import_pgdata::ImportingTimeline;
@@ -922,6 +929,124 @@ pub(crate) enum FinalizeTimelineImportError {
ShuttingDown,
}
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub(crate) struct TimelineOwnershipRecord {
+ schema_version: u8,
+ token: String,
+ request_hash: String,
+}
+
+async fn read_timeline_ownership_object(
+ remote_storage: &GenericRemoteStorage,
+ path: &remote_storage::RemotePath,
+ cancel: &CancellationToken,
+ kind: &'static str,
+) -> anyhow::Result<Option<TimelineOwnershipRecord>> {
+ let download = match remote_storage
+ .download(path, &DownloadOpts::default(), cancel)
+ .await
+ {
+ Ok(download) => download,
+ Err(DownloadError::NotFound) => return Ok(None),
+ Err(error) => return Err(error.into()),
+ };
+ let bytes = download
+ .download_stream
+ .try_fold(Vec::new(), |mut all, chunk| async move {
+ if all.len() + chunk.len() > 4096 {
+ return Err(std::io::Error::other(format!(
+ "timeline ownership {kind} exceeds limit"
+ )));
+ }
+ all.extend_from_slice(&chunk);
+ Ok(all)
+ })
+ .await?;
+ let record: TimelineOwnershipRecord = serde_json::from_slice(&bytes)?;
+ anyhow::ensure!(
+ record.schema_version == 1,
+ "unsupported timeline ownership {kind}"
+ );
+ Ok(Some(record))
+}
+
+pub(crate) async fn read_remote_timeline_ownership(
+ remote_storage: &GenericRemoteStorage,
+ tenant_shard_id: &TenantShardId,
+ timeline_id: &TimelineId,
+ cancel: &CancellationToken,
+) -> anyhow::Result<Option<TimelineOwnershipRecord>> {
+ let path = remote_timeline_client::remote_timeline_ownership_reservation_path(
+ tenant_shard_id,
+ timeline_id,
+ );
+ read_timeline_ownership_object(remote_storage, &path, cancel, "record").await
+}
+
+pub(crate) async fn read_remote_timeline_ownership_tombstone(
+ remote_storage: &GenericRemoteStorage,
+ tenant_shard_id: &TenantShardId,
+ timeline_id: &TimelineId,
+ cancel: &CancellationToken,
+) -> anyhow::Result<Option<TimelineOwnershipRecord>> {
+ let path = remote_timeline_client::remote_timeline_ownership_tombstone_path(
+ tenant_shard_id,
+ timeline_id,
+ );
+ read_timeline_ownership_object(remote_storage, &path, cancel, "tombstone").await
+}
+
+pub(crate) async fn tombstone_remote_timeline_ownership(
+ remote_storage: &GenericRemoteStorage,
+ tenant_shard_id: &TenantShardId,
+ timeline_id: &TimelineId,
+ token: &str,
+ cancel: &CancellationToken,
+) -> Result<(), ApiError> {
+ let record =
+ read_remote_timeline_ownership(remote_storage, tenant_shard_id, timeline_id, cancel)
+ .await
+ .map_err(ApiError::InternalServerError)?
+ .ok_or_else(|| {
+ ApiError::Conflict("timeline has no durable ownership record".to_string())
+ })?;
+ if record.token != token {
+ return Err(ApiError::Conflict(
+ "timeline deletion ownership token does not match".to_string(),
+ ));
+ }
+ let path = remote_timeline_client::remote_timeline_ownership_tombstone_path(
+ tenant_shard_id,
+ timeline_id,
+ );
+ let encoded =
+ serde_json::to_vec(&record).map_err(|error| ApiError::InternalServerError(error.into()))?;
+ match remote_storage
+ .upload_if_absent(Bytes::from(encoded), &path, cancel)
+ .await
+ {
+ Ok(()) => Ok(()),
+ Err(ConditionalCreateError::AlreadyExists) => {
+ let existing = read_remote_timeline_ownership_tombstone(
+ remote_storage,
+ tenant_shard_id,
+ timeline_id,
+ cancel,
+ )
+ .await
+ .map_err(ApiError::InternalServerError)?;
+ if existing.as_ref() == Some(&record) {
+ Ok(())
+ } else {
+ Err(ApiError::Conflict(
+ "timeline deletion tombstone conflicts with durable ownership".to_string(),
+ ))
+ }
+ }
+ Err(ConditionalCreateError::Other(error)) => Err(ApiError::InternalServerError(error)),
+ }
+}
+
/// Arguments to [`TenantShard::create_timeline`].
///
/// Not usable as an idempotency key for timeline creation because if [`CreateTimelineParamsBranch::ancestor_start_lsn`]
@@ -1151,6 +1276,202 @@ pub(crate) enum LoadConfigError {
}
impl TenantShard {
+ async fn ensure_tenant_delete_not_fenced(&self) -> Result<(), ApiError> {
+ let path = remote_timeline_client::remote_tenant_ownership_deletion_path(
+ &self.tenant_shard_id.tenant_id,
+ );
+ match self
+ .remote_storage
+ .download(&path, &DownloadOpts::default(), &self.cancel)
+ .await
+ {
+ Err(DownloadError::NotFound) => Ok(()),
+ Ok(_) => Err(ApiError::Conflict(
+ "tenant ownership is permanently deleting".to_string(),
+ )),
+ Err(error) => Err(ApiError::InternalServerError(error.into())),
+ }
+ }
+
+ async fn ensure_remote_timeline_prefix_empty(
+ &self,
+ timeline_id: TimelineId,
+ ) -> Result<(), ApiError> {
+ let prefix =
+ remote_timeline_client::remote_timeline_path(&self.tenant_shard_id, &timeline_id)
+ .add_trailing_slash();
+ let listing = self
+ .remote_storage
+ .list(
+ Some(&prefix),
+ ListingMode::NoDelimiter,
+ NonZeroU32::new(1),
+ &self.cancel,
+ )
+ .await
+ .map_err(|error| ApiError::InternalServerError(error.into()))?;
+ if !listing.keys.is_empty() || !listing.prefixes.is_empty() {
+ return Err(ApiError::Conflict(
+ "refusing to adopt a pre-existing remote timeline".to_string(),
+ ));
+ }
+ Ok(())
+ }
+
+ pub(crate) async fn reserve_timeline_ownership(
+ &self,
+ timeline_id: TimelineId,
+ ownership_token: &str,
+ ownership_request_hash: &str,
+ ) -> Result<(), ApiError> {
+ self.ensure_tenant_delete_not_fenced().await?;
+ if ownership_token.len() != 32
+ || ownership_token == "00000000000000000000000000000000"
+ || !ownership_token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ || ownership_request_hash.len() != 64
+ || !ownership_request_hash
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "invalid timeline ownership token or request hash"
+ )));
+ }
+ let wanted = TimelineOwnershipRecord {
+ schema_version: 1,
+ token: ownership_token.to_string(),
+ request_hash: ownership_request_hash.to_string(),
+ };
+ let tombstone_path = remote_timeline_client::remote_timeline_ownership_tombstone_path(
+ &self.tenant_shard_id,
+ &timeline_id,
+ );
+ match self
+ .remote_storage
+ .download(&tombstone_path, &DownloadOpts::default(), &self.cancel)
+ .await
+ {
+ Ok(_) => {
+ return Err(ApiError::Conflict(
+ "timeline ownership is permanently tombstoned".to_string(),
+ ));
+ }
+ Err(DownloadError::NotFound) => {}
+ Err(error) => return Err(ApiError::InternalServerError(error.into())),
+ }
+ let already_reserved = if let Some(stored) = self
+ .read_timeline_ownership(timeline_id)
+ .await
+ .map_err(ApiError::InternalServerError)?
+ {
+ if stored != wanted {
+ return Err(ApiError::Conflict(
+ "timeline ownership token or request hash conflicts with the durable record"
+ .to_string(),
+ ));
+ }
+ true
+ } else {
+ false
+ };
+ if !already_reserved {
+ if self.get_timeline(timeline_id, false).is_ok() {
+ return Err(ApiError::Conflict(
+ "refusing to adopt a pre-existing unowned timeline".to_string(),
+ ));
+ }
+ self.ensure_remote_timeline_prefix_empty(timeline_id)
+ .await?;
+ let encoded = serde_json::to_vec(&wanted)
+ .map_err(|error| ApiError::InternalServerError(error.into()))?;
+ let path = remote_timeline_client::remote_timeline_ownership_reservation_path(
+ &self.tenant_shard_id,
+ &timeline_id,
+ );
+ let reservation = match self
+ .remote_storage
+ .upload_if_absent(Bytes::from(encoded.clone()), &path, &self.cancel)
+ .await
+ {
+ Ok(()) => Ok(()),
+ Err(ConditionalCreateError::AlreadyExists) => {
+ let stored = self
+ .read_timeline_ownership(timeline_id)
+ .await
+ .map_err(ApiError::InternalServerError)?
+ .ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!(
+ "timeline ownership record disappeared after create conflict"
+ ))
+ })?;
+ if stored == wanted {
+ Ok(())
+ } else {
+ Err(ApiError::Conflict("timeline ownership token or request hash conflicts with the durable record".to_string()))
+ }
+ }
+ Err(ConditionalCreateError::Other(error)) => {
+ Err(ApiError::InternalServerError(error))
+ }
+ };
+ reservation?;
+ }
+ // The conditional fence is outside the data prefix. A fresh reservation
+ // rechecks the exact prefix after publication so an object racing the
+ // first list cannot be adopted. An exact replay is allowed to resume
+ // remote data written after that immutable reservation; its token and
+ // request hash are the authority for that interrupted creation.
+ if !already_reserved && self.get_timeline(timeline_id, false).is_err() {
+ self.ensure_remote_timeline_prefix_empty(timeline_id)
+ .await?;
+ }
+ self.ensure_tenant_delete_not_fenced().await?;
+ // Delete publishes a permanent marker in a sibling prefix. Recheck it
+ // after the conditional active-record create so concurrent delete can
+ // never leave a successful reservation behind a tombstone.
+ match self
+ .remote_storage
+ .download(&tombstone_path, &DownloadOpts::default(), &self.cancel)
+ .await
+ {
+ Ok(_) => Err(ApiError::Conflict(
+ "timeline ownership is permanently tombstoned".to_string(),
+ )),
+ Err(DownloadError::NotFound) => Ok(()),
+ Err(error) => Err(ApiError::InternalServerError(error.into())),
+ }
+ }
+
+ async fn read_timeline_ownership(
+ &self,
+ timeline_id: TimelineId,
+ ) -> anyhow::Result<Option<TimelineOwnershipRecord>> {
+ read_remote_timeline_ownership(
+ &self.remote_storage,
+ &self.tenant_shard_id,
+ &timeline_id,
+ &self.cancel,
+ )
+ .await
+ }
+
+ pub(crate) async fn tombstone_timeline_ownership(
+ &self,
+ timeline_id: TimelineId,
+ token: &str,
+ ) -> Result<(), ApiError> {
+ tombstone_remote_timeline_ownership(
+ &self.remote_storage,
+ &self.tenant_shard_id,
+ &timeline_id,
+ token,
+ &self.cancel,
+ )
+ .await
+ }
+
/// Yet another helper for timeline initialization.
///
/// - Initializes the Timeline struct and inserts it into the tenant's hash map
@@ -1174,6 +1495,17 @@ async fn timeline_init_and_sync(
) -> anyhow::Result<TimelineInitAndSyncResult> {
let tenant_id = self.tenant_shard_id;
+ let ownership = self.read_timeline_ownership(timeline_id).await?;
+ if self.conf.hakopod_ownership_v1 || index_part.ownership_token.is_some() {
+ let ownership = ownership.context("timeline ownership record is missing")?;
+ anyhow::ensure!(
+ index_part.ownership_token.as_deref() == Some(ownership.token.as_str())
+ && index_part.ownership_request_hash.as_deref()
+ == Some(ownership.request_hash.as_str()),
+ "timeline ownership record does not match remote index"
+ );
+ }
+
let import_pgdata = index_part.import_pgdata.clone();
let idempotency = match &import_pgdata {
Some(import_pgdata) => {
@@ -5475,11 +5807,18 @@ async fn prepare_new_timeline<'a>(
) -> anyhow::Result<(UninitializedTimeline<'a>, RequestContext)> {
let tenant_shard_id = self.tenant_shard_id;
+ let ownership = self.read_timeline_ownership(new_timeline_id).await?;
+ if self.conf.hakopod_ownership_v1 {
+ anyhow::ensure!(ownership.is_some(), "timeline ownership record is missing");
+ }
+
let resources = self.build_timeline_resources(new_timeline_id);
resources.remote_client.init_upload_queue_for_empty_remote(
new_metadata,
rel_size_v2_status.clone(),
rel_size_migrated_at,
+ ownership.as_ref().map(|value| value.token.clone()),
+ ownership.map(|value| value.request_hash),
)?;
let (timeline_struct, timeline_ctx) = self
diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs
index 0feba5e9c88aaae53f513925a2dd9acf0be7a1cd..5920bdd096cb4ccff8bd70151fe4bd34afc66d54 100644
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -4,23 +4,30 @@
use std::borrow::Cow;
use std::cmp::Ordering;
use std::collections::{BTreeMap, HashMap, HashSet};
+use std::num::NonZeroU32;
use std::ops::Deref;
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
+use bytes::Bytes;
use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf};
-use futures::StreamExt;
+use futures::{StreamExt, TryStreamExt};
use itertools::Itertools;
use pageserver_api::key::Key;
-use pageserver_api::models::{DetachBehavior, LocationConfigMode};
+use pageserver_api::models::{
+ DetachBehavior, LocationConfigMode, OwnedTimelineDelete, TenantDeletePreflightResponse,
+ TenantDeleteRequest,
+};
use pageserver_api::shard::{
ShardCount, ShardIdentity, ShardIndex, ShardNumber, ShardStripeSize, TenantShardId,
};
use pageserver_api::upcall_api::ReAttachResponseTenant;
use rand::Rng;
use rand::distr::Alphanumeric;
-use remote_storage::TimeoutOrCancel;
+use remote_storage::{ConditionalCreateError, DownloadError, DownloadOpts, TimeoutOrCancel};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
use sysinfo::SystemExt;
use tokio::fs;
use tokio::task::JoinSet;
@@ -32,7 +39,10 @@
use utils::id::{TenantId, TimelineId};
use utils::{backoff, completion, crashsafe};
-use super::remote_timeline_client::remote_tenant_path;
+use super::remote_timeline_client::{
+ list_remote_tenant_shards_bounded, list_remote_timelines_bounded,
+ remote_tenant_ownership_deletion_path, remote_tenant_path, remote_timeline_path,
+};
use super::secondary::SecondaryTenant;
use super::timeline::detach_ancestor::{self, PreparedTimelineDetach};
use super::{GlobalShutDown, TenantSharedResources};
@@ -475,6 +485,9 @@ pub(crate) enum DeleteTenantError {
#[error("Tenant map slot error {0}")]
SlotError(#[from] TenantSlotError),
+ #[error("Tenant ownership conflict: {0}")]
+ OwnershipConflict(String),
+
#[error("Cancelled")]
Cancelled,
@@ -770,6 +783,20 @@ pub(crate) enum UpsertLocationError {
InternalError(anyhow::Error),
}
+struct OwnedTenantDeletePlan {
+ digest: String,
+ authorization_digest: String,
+ timeline_paths: Vec<remote_storage::RemotePath>,
+ tombstones: Vec<(TenantShardId, TimelineId, String)>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+struct TenantDeleteFence {
+ schema_version: u8,
+ delete_token: String,
+ authorization_digest: String,
+}
+
impl TenantManager {
/// Convenience function so that anyone with a TenantManager can get at the global configuration, without
/// having to pass it around everywhere as a separate object.
@@ -1352,6 +1379,320 @@ pub(crate) fn get(&self, tenant_shard_id: TenantShardId) -> Option<TenantSlot> {
}
}
+ async fn validate_owned_tenant_delete(
+ &self,
+ tenant_id: TenantId,
+ owned_timelines: &[OwnedTimelineDelete],
+ ) -> Result<OwnedTenantDeletePlan, DeleteTenantError> {
+ const MAX_OWNED_TIMELINES: usize = 4096;
+ let listing_limit = NonZeroU32::new((MAX_OWNED_TIMELINES + 1) as u32).unwrap();
+ if owned_timelines.len() > MAX_OWNED_TIMELINES {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "owned timeline authorization count exceeds limit of {MAX_OWNED_TIMELINES}"
+ )));
+ }
+
+ let mut supplied = BTreeMap::new();
+ for owned_timeline in owned_timelines {
+ let token = &owned_timeline.ownership_token;
+ let request_hash = &owned_timeline.ownership_request_hash;
+ if token.len() != 32
+ || token == "00000000000000000000000000000000"
+ || !token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ || request_hash.len() != 64
+ || !request_hash
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ || supplied
+ .insert(
+ owned_timeline.timeline_id,
+ (token.clone(), request_hash.clone()),
+ )
+ .is_some()
+ {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "invalid or duplicate owned timeline authorization".to_string(),
+ ));
+ }
+ }
+
+ let (tenant_shards, other_tenant_keys) = list_remote_tenant_shards_bounded(
+ &self.resources.remote_storage,
+ tenant_id,
+ listing_limit,
+ self.cancel.clone(),
+ )
+ .await?;
+ let tenant_shards = tenant_shards.into_iter().sorted().collect::<Vec<_>>();
+ if tenant_shards.len() + other_tenant_keys.len() > MAX_OWNED_TIMELINES {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "tenant remote shard count exceeds limit of {MAX_OWNED_TIMELINES}"
+ )));
+ }
+ if !other_tenant_keys.is_empty() {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "tenant has unrecognized remote descendants: {other_tenant_keys:?}"
+ )));
+ }
+
+ let mut authorization_hasher = Sha256::new();
+ authorization_hasher.update(b"hakopod-pageserver-tenant-delete-authorization-v1\0");
+ authorization_hasher.update(tenant_id.to_string().as_bytes());
+ authorization_hasher.update(b"\0");
+ for (timeline_id, (token, request_hash)) in &supplied {
+ authorization_hasher.update(timeline_id.to_string().as_bytes());
+ authorization_hasher.update(b"\0");
+ authorization_hasher.update(token.as_bytes());
+ authorization_hasher.update(b"\0");
+ authorization_hasher.update(request_hash.as_bytes());
+ authorization_hasher.update(b"\0");
+ }
+ let authorization_digest = hex::encode(authorization_hasher.finalize());
+
+ let mut hasher = Sha256::new();
+ hasher.update(b"hakopod-pageserver-tenant-delete-preflight-v1\0");
+ hasher.update(tenant_id.to_string().as_bytes());
+ hasher.update(b"\0");
+ for (timeline_id, (token, request_hash)) in &supplied {
+ let unsharded = TenantShardId::unsharded(tenant_id);
+ let record = super::read_remote_timeline_ownership(
+ &self.resources.remote_storage,
+ &unsharded,
+ timeline_id,
+ &self.cancel,
+ )
+ .await?
+ .ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(format!(
+ "timeline {tenant_id}/{timeline_id} has no durable ownership reservation"
+ ))
+ })?;
+ if record.token != *token || record.request_hash != *request_hash {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "timeline {tenant_id}/{timeline_id} ownership reservation does not match"
+ )));
+ }
+ let tombstone = super::read_remote_timeline_ownership_tombstone(
+ &self.resources.remote_storage,
+ &unsharded,
+ timeline_id,
+ &self.cancel,
+ )
+ .await?;
+ if tombstone.as_ref().is_some_and(|value| value != &record) {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "timeline {tenant_id}/{timeline_id} ownership tombstone does not match"
+ )));
+ }
+ hasher.update(b"authorized\0");
+ hasher.update(timeline_id.to_string().as_bytes());
+ hasher.update(b"\0");
+ hasher.update(token.as_bytes());
+ hasher.update(b"\0");
+ hasher.update(request_hash.as_bytes());
+ hasher.update(if tombstone.is_some() {
+ b"\0tombstoned\0".as_slice()
+ } else {
+ b"\0active\0".as_slice()
+ });
+ }
+
+ let mut checked = 0usize;
+ let mut timeline_paths = Vec::new();
+ let mut tombstones = Vec::new();
+ let mut ownership_records = HashMap::new();
+ for remote_tenant_shard_id in tenant_shards {
+ hasher.update(b"shard\0");
+ hasher.update(remote_tenant_shard_id.to_string().as_bytes());
+ hasher.update(b"\0");
+ let (timeline_ids, other_timeline_keys) = list_remote_timelines_bounded(
+ &self.resources.remote_storage,
+ remote_tenant_shard_id,
+ listing_limit,
+ self.cancel.clone(),
+ )
+ .await?;
+ let timeline_ids = timeline_ids.into_iter().sorted().collect::<Vec<_>>();
+ if !other_timeline_keys.is_empty() {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "tenant shard {remote_tenant_shard_id} has unrecognized timeline descendants: {other_timeline_keys:?}"
+ )));
+ }
+ checked = checked.checked_add(timeline_ids.len()).ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(
+ "owned timeline count exceeds limit".to_string(),
+ )
+ })?;
+ if checked > MAX_OWNED_TIMELINES {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "owned timeline count exceeds limit of {MAX_OWNED_TIMELINES}"
+ )));
+ }
+
+ for timeline_id in timeline_ids {
+ let record = super::read_remote_timeline_ownership(
+ &self.resources.remote_storage,
+ &remote_tenant_shard_id,
+ &timeline_id,
+ &self.cancel,
+ )
+ .await?
+ .ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(format!(
+ "timeline {remote_tenant_shard_id}/{timeline_id} has no durable ownership record"
+ ))
+ })?;
+ if let Some(existing) = ownership_records.insert(timeline_id, record.clone()) {
+ if existing != record {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "timeline {timeline_id} ownership records disagree across tenant shards"
+ )));
+ }
+ }
+ let (token, request_hash) = supplied.get(&timeline_id).ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(format!(
+ "timeline {remote_tenant_shard_id}/{timeline_id} has no owned deletion authorization"
+ ))
+ })?;
+ if record.token != *token || record.request_hash != *request_hash {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "timeline {remote_tenant_shard_id}/{timeline_id} ownership does not match"
+ )));
+ }
+ let tombstone = super::read_remote_timeline_ownership_tombstone(
+ &self.resources.remote_storage,
+ &remote_tenant_shard_id,
+ &timeline_id,
+ &self.cancel,
+ )
+ .await?;
+ let tombstone_state = match tombstone {
+ Some(tombstone) if record == tombstone => "tombstoned",
+ Some(_) => {
+ return Err(DeleteTenantError::OwnershipConflict(format!(
+ "timeline {remote_tenant_shard_id}/{timeline_id} ownership tombstone does not match"
+ )));
+ }
+ None => {
+ tombstones.push((remote_tenant_shard_id, timeline_id, token.clone()));
+ "active"
+ }
+ };
+ hasher.update(b"timeline\0");
+ hasher.update(remote_tenant_shard_id.to_string().as_bytes());
+ hasher.update(b"/");
+ hasher.update(timeline_id.to_string().as_bytes());
+ hasher.update(b"\0");
+ hasher.update(token.as_bytes());
+ hasher.update(b"\0");
+ hasher.update(request_hash.as_bytes());
+ hasher.update(b"\0");
+ hasher.update(tombstone_state.as_bytes());
+ hasher.update(b"\0");
+ timeline_paths.push(remote_timeline_path(&remote_tenant_shard_id, &timeline_id));
+ }
+ }
+
+ Ok(OwnedTenantDeletePlan {
+ digest: hex::encode(hasher.finalize()),
+ authorization_digest,
+ timeline_paths,
+ tombstones,
+ })
+ }
+
+ async fn install_tenant_delete_fence(
+ &self,
+ tenant_id: TenantId,
+ authorization_digest: &str,
+ ) -> Result<TenantDeleteFence, DeleteTenantError> {
+ let fence = TenantDeleteFence {
+ schema_version: 1,
+ delete_token: uuid::Uuid::new_v4().simple().to_string(),
+ authorization_digest: authorization_digest.to_string(),
+ };
+ let encoded = serde_json::to_vec(&fence).map_err(|error| {
+ DeleteTenantError::Other(anyhow::anyhow!("encode tenant delete fence: {error}"))
+ })?;
+ let path = remote_tenant_ownership_deletion_path(&tenant_id);
+ match self
+ .resources
+ .remote_storage
+ .upload_if_absent(Bytes::from(encoded), &path, &self.cancel)
+ .await
+ {
+ Ok(()) => Ok(fence),
+ Err(ConditionalCreateError::AlreadyExists) => {
+ let download = self
+ .resources
+ .remote_storage
+ .download(&path, &DownloadOpts::default(), &self.cancel)
+ .await
+ .map_err(|error| DeleteTenantError::Other(error.into()))?;
+ let bytes = download
+ .download_stream
+ .try_fold(Vec::new(), |mut all, chunk| async move {
+ if all.len() + chunk.len() > 4096 {
+ return Err(std::io::Error::other("tenant delete fence exceeds limit"));
+ }
+ all.extend_from_slice(&chunk);
+ Ok(all)
+ })
+ .await
+ .map_err(|error| DeleteTenantError::Other(error.into()))?;
+ let existing: TenantDeleteFence = serde_json::from_slice(&bytes)
+ .map_err(|error| DeleteTenantError::Other(error.into()))?;
+ if existing.schema_version == 1
+ && existing.authorization_digest == authorization_digest
+ && existing.delete_token.len() == 32
+ && existing
+ .delete_token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ Ok(existing)
+ } else {
+ Err(DeleteTenantError::OwnershipConflict(
+ "tenant delete fence conflicts with the authorized resources".to_string(),
+ ))
+ }
+ }
+ Err(ConditionalCreateError::Other(error)) => Err(DeleteTenantError::Other(error)),
+ }
+ }
+
+ async fn read_tenant_delete_fence(
+ &self,
+ tenant_id: TenantId,
+ ) -> Result<TenantDeleteFence, DeleteTenantError> {
+ let path = remote_tenant_ownership_deletion_path(&tenant_id);
+ let download = self
+ .resources
+ .remote_storage
+ .download(&path, &DownloadOpts::default(), &self.cancel)
+ .await
+ .map_err(|error| match error {
+ DownloadError::NotFound => DeleteTenantError::OwnershipConflict(
+ "tenant deletion has not been prepared".to_string(),
+ ),
+ error => DeleteTenantError::Other(error.into()),
+ })?;
+ let bytes = download
+ .download_stream
+ .try_fold(Vec::new(), |mut all, chunk| async move {
+ if all.len() + chunk.len() > 4096 {
+ return Err(std::io::Error::other("tenant delete fence exceeds limit"));
+ }
+ all.extend_from_slice(&chunk);
+ Ok(all)
+ })
+ .await
+ .map_err(|error| DeleteTenantError::Other(error.into()))?;
+ serde_json::from_slice(&bytes).map_err(|error| DeleteTenantError::Other(error.into()))
+ }
+
/// If a tenant is attached, detach it. Then remove its data from remote storage.
///
/// A tenant is considered deleted once it is gone from remote storage. It is the caller's
@@ -1366,7 +1707,8 @@ pub(crate) fn get(&self, tenant_shard_id: TenantShardId) -> Option<TenantSlot> {
pub(crate) async fn delete_tenant(
&self,
tenant_shard_id: TenantShardId,
- ) -> Result<(), DeleteTenantError> {
+ ownership: Option<TenantDeleteRequest>,
+ ) -> Result<Option<TenantDeletePreflightResponse>, DeleteTenantError> {
super::span::debug_assert_current_span_has_tenant_id();
async fn delete_local(
@@ -1384,8 +1726,124 @@ async fn delete_local(
Ok(())
}
+ // Strict-mode validation is read-only: it must not acquire a tenant slot
+ // or change local placement. Preparation publishes an immutable remote
+ // fence and returns its retry token, but does not remove tenant data.
+ if let Some(ownership) = ownership {
+ if ownership.validate_only && ownership.prepare_delete {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant deletion cannot validate and prepare in one request".to_string(),
+ ));
+ }
+ let plan = self
+ .validate_owned_tenant_delete(tenant_shard_id.tenant_id, &ownership.owned_timelines)
+ .await?;
+ if ownership.validate_only {
+ return Ok(Some(TenantDeletePreflightResponse {
+ schema_version: 1,
+ digest: plan.digest,
+ delete_token: None,
+ }));
+ }
+ if ownership.prepare_delete {
+ let expected = ownership.expected_preflight_digest.ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(
+ "tenant deletion preparation requires a preflight digest".to_string(),
+ )
+ })?;
+ if ownership.delete_token.is_some() {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant deletion preparation does not accept a caller token".to_string(),
+ ));
+ }
+ if expected != plan.digest {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant deletion preflight digest is stale".to_string(),
+ ));
+ }
+ let fence = self
+ .install_tenant_delete_fence(
+ tenant_shard_id.tenant_id,
+ &plan.authorization_digest,
+ )
+ .await?;
+ let fenced_plan = self
+ .validate_owned_tenant_delete(
+ tenant_shard_id.tenant_id,
+ &ownership.owned_timelines,
+ )
+ .await?;
+ if fenced_plan.digest != expected
+ || fenced_plan.authorization_digest != fence.authorization_digest
+ {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant ownership changed while deletion was fenced".to_string(),
+ ));
+ }
+ return Ok(Some(TenantDeletePreflightResponse {
+ schema_version: 1,
+ digest: fenced_plan.digest,
+ delete_token: Some(fence.delete_token),
+ }));
+ }
+
+ let supplied_delete_token = ownership.delete_token.as_deref().ok_or_else(|| {
+ DeleteTenantError::OwnershipConflict(
+ "tenant deletion requires its prepared token".to_string(),
+ )
+ })?;
+ let fence = self
+ .read_tenant_delete_fence(tenant_shard_id.tenant_id)
+ .await?;
+ if fence.schema_version != 1
+ || fence.delete_token != supplied_delete_token
+ || fence.authorization_digest != plan.authorization_digest
+ {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant deletion token or authorized resources do not match the prepared fence"
+ .to_string(),
+ ));
+ }
+
+ for (remote_tenant_shard_id, timeline_id, token) in plan.tombstones {
+ super::tombstone_remote_timeline_ownership(
+ &self.resources.remote_storage,
+ &remote_tenant_shard_id,
+ &timeline_id,
+ &token,
+ &self.cancel,
+ )
+ .await
+ .map_err(|error| match error {
+ http_utils::error::ApiError::Conflict(message) => {
+ DeleteTenantError::OwnershipConflict(message)
+ }
+ error => DeleteTenantError::Other(anyhow::anyhow!(error)),
+ })?;
+ }
+ for timeline_path in plan.timeline_paths {
+ self.resources
+ .remote_storage
+ .delete_prefix(&timeline_path, &self.cancel)
+ .await?;
+ }
+
+ let recheck = self
+ .validate_owned_tenant_delete(tenant_shard_id.tenant_id, &ownership.owned_timelines)
+ .await?;
+ // Every remote timeline prefix was removed. A new descendant here
+ // means a writer bypassed or raced the tenant fence, so fail before
+ // the tenant-wide sweep can erase it.
+ if !recheck.timeline_paths.is_empty() {
+ return Err(DeleteTenantError::OwnershipConflict(
+ "tenant still has remote timeline descendants after owned deletion".to_string(),
+ ));
+ }
+ }
+
let slot_guard =
self.tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any)?;
+
match &slot_guard.old_value {
Some(TenantSlot::Attached(tenant)) => {
// Legacy deletion flow: the tenant remains attached, goes to Stopping state, and
@@ -1437,7 +1895,8 @@ async fn delete_local(
return DeleteTenantError::Cancelled;
}
DeleteTenantError::Other(err)
- })
+ })?;
+ Ok(None)
}
#[instrument(skip_all, fields(tenant_id=%tenant.get_tenant_shard_id().tenant_id, shard_id=%tenant.get_tenant_shard_id().shard_slug(), new_shard_count=%new_shard_count.literal()))]
diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs
index 6b650beb3f3e41090ff74ef13f802d6e84a63927..c93fa05734bf3b37c566612b49dca82e4d69c0fb 100644
--- a/pageserver/src/tenant/remote_timeline_client.rs
+++ b/pageserver/src/tenant/remote_timeline_client.rs
@@ -190,7 +190,8 @@
use chrono::{NaiveDateTime, Utc};
pub(crate) use download::{
download_index_part, download_initdb_tar_zst, download_tenant_manifest, is_temp_download_file,
- list_remote_tenant_shards, list_remote_timelines,
+ list_remote_tenant_shards, list_remote_tenant_shards_bounded, list_remote_timelines,
+ list_remote_timelines_bounded,
};
use index::GcCompactionState;
pub(crate) use index::LayerFileMetadata;
@@ -376,6 +377,16 @@ fn drop(&mut self) {
}
impl RemoteTimelineClient {
+ pub(crate) fn ownership(&self) -> (Option<String>, Option<String>) {
+ let queue = self.upload_queue.lock().unwrap();
+ match &*queue {
+ UploadQueue::Initialized(queue) => (
+ queue.dirty.ownership_token.clone(),
+ queue.dirty.ownership_request_hash.clone(),
+ ),
+ UploadQueue::Uninitialized | UploadQueue::Stopped(_) => (None, None),
+ }
+ }
///
/// Create a remote storage client for given timeline
///
@@ -445,6 +456,8 @@ pub fn init_upload_queue_for_empty_remote(
local_metadata: &TimelineMetadata,
rel_size_v2_migration: Option<RelSizeMigration>,
rel_size_migrated_at: Option<Lsn>,
+ ownership_token: Option<String>,
+ ownership_request_hash: Option<String>,
) -> anyhow::Result<()> {
// Set the maximum number of inprogress tasks to the remote storage concurrency. There's
// certainly no point in starting more upload tasks than this.
@@ -456,6 +469,8 @@ pub fn init_upload_queue_for_empty_remote(
let mut upload_queue = self.upload_queue.lock().unwrap();
let initialized_queue =
upload_queue.initialize_empty_remote(local_metadata, inprogress_limit)?;
+ initialized_queue.dirty.ownership_token = ownership_token;
+ initialized_queue.dirty.ownership_request_hash = ownership_request_hash;
initialized_queue.dirty.rel_size_migration = rel_size_v2_migration;
initialized_queue.dirty.rel_size_migrated_at = rel_size_migrated_at;
self.update_remote_physical_size_gauge(None);
@@ -2680,6 +2695,40 @@ pub fn remote_timeline_path(
remote_timelines_path(tenant_shard_id).join(Utf8Path::new(&timeline_id.to_string()))
}
+/// Permanent ownership deletion marker. This deliberately lives outside the
+/// timeline prefix because timeline deletion sweeps every object in that
+/// prefix before removing the index.
+pub fn remote_timeline_ownership_tombstone_path(
+ tenant_shard_id: &TenantShardId,
+ timeline_id: &TimelineId,
+) -> RemotePath {
+ RemotePath::from_string("hakopod-ownership-v1/timeline-tombstones")
+ .expect("static ownership tombstone prefix is valid")
+ .join(Utf8Path::new(&tenant_shard_id.tenant_id.to_string()))
+ .join(Utf8Path::new(&format!("{timeline_id}.tombstone.json")))
+}
+
+/// Immutable ownership reservation for a timeline. This deliberately lives
+/// outside the timeline's data prefix, so checking or deleting timeline data
+/// cannot hide or remove the fence that authorizes strict-mode writers.
+pub fn remote_timeline_ownership_reservation_path(
+ tenant_shard_id: &TenantShardId,
+ timeline_id: &TimelineId,
+) -> RemotePath {
+ RemotePath::from_string("hakopod-ownership-v1/timeline-reservations")
+ .expect("static ownership reservation prefix is valid")
+ .join(Utf8Path::new(&tenant_shard_id.tenant_id.to_string()))
+ .join(Utf8Path::new(&format!("{timeline_id}.reservation.json")))
+}
+
+/// Immutable fence created when strict-mode tenant deletion begins. It is
+/// outside the tenant prefix so the final tenant sweep cannot remove it.
+pub fn remote_tenant_ownership_deletion_path(tenant_id: &TenantId) -> RemotePath {
+ RemotePath::from_string("hakopod-ownership-v1/tenant-deletions")
+ .expect("static tenant deletion prefix is valid")
+ .join(Utf8Path::new(&format!("{tenant_id}.json")))
+}
+
/// Obtains the path of the given Layer in the remote
///
/// Note that the shard component of a remote layer path is _not_ always the same
@@ -2797,6 +2846,45 @@ pub(super) fn dummy_contents(name: &str) -> Vec<u8> {
format!("contents for {name}").into()
}
+ #[test]
+ fn ownership_tombstone_survives_tenant_prefix_deletion() {
+ let tenant_id: TenantId = "0123456789abcdef0123456789abcdef".parse().unwrap();
+ let timeline_id: TimelineId = "fedcba9876543210fedcba9876543210".parse().unwrap();
+ let tenant_shard_id = TenantShardId::unsharded(tenant_id);
+ let tenant_path = remote_tenant_path(&tenant_shard_id);
+ let tombstone_path =
+ remote_timeline_ownership_tombstone_path(&tenant_shard_id, &timeline_id);
+ let reservation_path =
+ remote_timeline_ownership_reservation_path(&tenant_shard_id, &timeline_id);
+ let delete_fence_path = remote_tenant_ownership_deletion_path(&tenant_id);
+
+ for durable_path in [&tombstone_path, &reservation_path, &delete_fence_path] {
+ assert!(
+ durable_path
+ .get_path()
+ .strip_prefix(tenant_path.get_path())
+ .is_err(),
+ "tenant prefix deletion must not remove ownership fences"
+ );
+ }
+ assert_eq!(
+ tombstone_path.to_string(),
+ format!(
+ "hakopod-ownership-v1/timeline-tombstones/{tenant_id}/{timeline_id}.tombstone.json"
+ )
+ );
+ assert_eq!(
+ reservation_path.to_string(),
+ format!(
+ "hakopod-ownership-v1/timeline-reservations/{tenant_id}/{timeline_id}.reservation.json"
+ )
+ );
+ assert_eq!(
+ delete_fence_path.to_string(),
+ format!("hakopod-ownership-v1/tenant-deletions/{tenant_id}.json")
+ );
+ }
+
pub(super) fn dummy_metadata(disk_consistent_lsn: Lsn) -> TimelineMetadata {
let metadata = TimelineMetadata::new(
disk_consistent_lsn,
diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs
index 84989e0fb8f2477dbd4871263849cecb934bcf2b..1d94edc8d40c0da17233c52fc370210adec71e7d 100644
--- a/pageserver/src/tenant/remote_timeline_client/download.rs
+++ b/pageserver/src/tenant/remote_timeline_client/download.rs
@@ -5,6 +5,7 @@
use std::collections::HashSet;
use std::future::Future;
+use std::num::NonZeroU32;
use std::str::FromStr;
use std::sync::atomic::AtomicU64;
use std::time::SystemTime;
@@ -255,13 +256,14 @@ pub(crate) fn is_temp_download_file(path: &Utf8Path) -> bool {
async fn list_identifiers<T>(
storage: &GenericRemoteStorage,
prefix: RemotePath,
+ max_keys: Option<NonZeroU32>,
cancel: CancellationToken,
) -> anyhow::Result<(HashSet<T>, HashSet<String>)>
where
T: FromStr + Eq + std::hash::Hash,
{
let listing = download_retry_forever(
- || storage.list(Some(&prefix), ListingMode::WithDelimiter, None, &cancel),
+ || storage.list(Some(&prefix), ListingMode::WithDelimiter, max_keys, &cancel),
&format!("list identifiers in prefix {prefix}"),
&cancel,
)
@@ -299,7 +301,17 @@ pub(crate) async fn list_remote_tenant_shards(
cancel: CancellationToken,
) -> anyhow::Result<(HashSet<TenantShardId>, HashSet<String>)> {
let remote_path = remote_tenant_path(&TenantShardId::unsharded(tenant_id));
- list_identifiers::<TenantShardId>(storage, remote_path, cancel).await
+ list_identifiers::<TenantShardId>(storage, remote_path, None, cancel).await
+}
+
+pub(crate) async fn list_remote_tenant_shards_bounded(
+ storage: &GenericRemoteStorage,
+ tenant_id: TenantId,
+ max_keys: NonZeroU32,
+ cancel: CancellationToken,
+) -> anyhow::Result<(HashSet<TenantShardId>, HashSet<String>)> {
+ let remote_path = remote_tenant_path(&TenantShardId::unsharded(tenant_id));
+ list_identifiers::<TenantShardId>(storage, remote_path, Some(max_keys), cancel).await
}
/// List timelines of given tenant shard in remote storage
@@ -313,7 +325,17 @@ pub async fn list_remote_timelines(
});
let remote_path = remote_timelines_path(&tenant_shard_id).add_trailing_slash();
- list_identifiers::<TimelineId>(storage, remote_path, cancel).await
+ list_identifiers::<TimelineId>(storage, remote_path, None, cancel).await
+}
+
+pub(crate) async fn list_remote_timelines_bounded(
+ storage: &GenericRemoteStorage,
+ tenant_shard_id: TenantShardId,
+ max_keys: NonZeroU32,
+ cancel: CancellationToken,
+) -> anyhow::Result<(HashSet<TimelineId>, HashSet<String>)> {
+ let remote_path = remote_timelines_path(&tenant_shard_id).add_trailing_slash();
+ list_identifiers::<TimelineId>(storage, remote_path, Some(max_keys), cancel).await
}
async fn do_download_remote_path_retry_forever(
diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs
index 9531e7f650880253564ea0787cc143d7f554bc81..3500f813cdbdd37d34ac50c29bb5384f8cac8831 100644
--- a/pageserver/src/tenant/remote_timeline_client/index.rs
+++ b/pageserver/src/tenant/remote_timeline_client/index.rs
@@ -31,6 +31,11 @@ pub struct IndexPart {
#[serde(default)]
version: usize,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) ownership_token: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub(crate) ownership_request_hash: Option<String>,
+
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
pub deleted_at: Option<NaiveDateTime>,
@@ -148,17 +153,19 @@ impl IndexPart {
/// - 13: +gc_compaction
/// - 14: +marked_invisible_at
/// - 15: +rel_size_migrated_at
- const LATEST_VERSION: usize = 15;
+ const LATEST_VERSION: usize = 16;
// Versions we may see when reading from a bucket.
pub const KNOWN_VERSIONS: &'static [usize] =
- &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
+ &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
pub const FILE_NAME: &'static str = "index_part.json";
pub fn empty(metadata: TimelineMetadata) -> Self {
IndexPart {
version: Self::LATEST_VERSION,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: Default::default(),
disk_consistent_lsn: metadata.disk_consistent_lsn(),
metadata,
@@ -457,6 +464,8 @@ fn v1_indexpart_is_parsed() {
let expected = IndexPart {
// note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
version: 1,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -507,6 +516,8 @@ fn v1_indexpart_is_parsed_with_optional_missing_layers() {
let expected = IndexPart {
// note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
version: 1,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -558,6 +569,8 @@ fn v2_indexpart_is_parsed_with_deleted_at() {
let expected = IndexPart {
// note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead?
version: 2,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -603,6 +616,8 @@ fn empty_layers_are_parsed() {
let expected = IndexPart {
version: 1,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::new(),
disk_consistent_lsn: "0/2532648".parse::<Lsn>().unwrap(),
metadata: TimelineMetadata::from_bytes(&[
@@ -661,6 +676,8 @@ fn v4_indexpart_is_parsed() {
let expected = IndexPart {
version: 4,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -711,6 +728,8 @@ fn v5_indexpart_is_parsed() {
let expected = IndexPart {
version: 5,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000014EF420-00000000014EF499".parse().unwrap(), LayerFileMetadata {
file_size: 23289856,
@@ -766,6 +785,8 @@ fn v6_indexpart_is_parsed() {
let expected = IndexPart {
version: 6,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -826,6 +847,8 @@ fn v7_indexpart_is_parsed() {
let expected = IndexPart {
version: 7,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -889,6 +912,8 @@ fn v8_indexpart_is_parsed() {
let expected = IndexPart {
version: 8,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -954,6 +979,8 @@ fn v9_indexpart_is_parsed() {
let expected = IndexPart {
version: 9,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1031,6 +1058,8 @@ fn v10_importpgdata_is_parsed() {
let expected = IndexPart {
version: 10,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1113,6 +1142,8 @@ fn v11_rel_size_migration_is_parsed() {
let expected = IndexPart {
version: 11,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1199,6 +1230,8 @@ fn v12_v13_l2_gc_ompaction_is_parsed() {
let expected = IndexPart {
version: 13,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1288,6 +1321,8 @@ fn v14_marked_invisible_at_is_parsed() {
let expected = IndexPart {
version: 14,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1378,6 +1413,8 @@ fn v15_rel_size_migrated_at_is_parsed() {
let expected = IndexPart {
version: 15,
+ ownership_token: None,
+ ownership_request_hash: None,
layer_metadata: HashMap::from([
("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), LayerFileMetadata {
file_size: 25600000,
@@ -1429,4 +1466,15 @@ fn v15_rel_size_migrated_at_is_parsed() {
fn parse_naive_datetime(s: &str) -> NaiveDateTime {
chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%f").unwrap()
}
+
+ #[test]
+ fn ownership_survives_index_round_trip() {
+ let mut part = IndexPart::empty(TimelineMetadata::example());
+ part.ownership_token = Some("intent-12345678".to_string());
+ part.ownership_request_hash = Some("a".repeat(64));
+ let bytes = serde_json::to_vec(&part).unwrap();
+ let restored = IndexPart::from_json_bytes(&bytes).unwrap();
+ assert_eq!(restored.ownership_token, part.ownership_token);
+ assert_eq!(restored.ownership_request_hash, part.ownership_request_hash);
+ }
}
diff --git a/proxy/src/binary/proxy.rs b/proxy/src/binary/proxy.rs
index 86c84ed129e92c054cd6426efe03ce9cddd1caee..ab768ab758860f71b8d4a32fef3df0bca2632361 100644
--- a/proxy/src/binary/proxy.rs
+++ b/proxy/src/binary/proxy.rs
@@ -814,10 +814,8 @@ fn build_auth_backend(
let url: crate::url::ApiUrl = args.auth_endpoint.parse()?;
- let endpoint = http::Endpoint::new(
- url,
- http::new_control_plane_client(Duration::from_secs(2)),
- );
+ let endpoint =
+ http::Endpoint::new(url, http::new_control_plane_client(Duration::from_secs(2)));
let mut wake_compute_rps_limit = args.wake_compute_limit.clone();
RateBucketInfo::validate(&mut wake_compute_rps_limit)?;
diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml
index 56822b5c25d89b615ea832e3524bf4685a029ca7..964f5818e1991b004d2155125972232a36b872a6 100644
--- a/safekeeper/Cargo.toml
+++ b/safekeeper/Cargo.toml
@@ -72,7 +72,7 @@ http-utils.workspace = true
utils.workspace = true
wal_decoder.workspace = true
env_logger.workspace = true
-nix.workspace = true
+nix = { workspace = true, features = ["user"] }
workspace_hack.workspace = true
diff --git a/safekeeper/client/src/mgmt_api.rs b/safekeeper/client/src/mgmt_api.rs
index 3c8db3029eb59b852aec3c74128ed3530bc3476c..3307259a63b86fdaca065e004f4a3e593379035e 100644
--- a/safekeeper/client/src/mgmt_api.rs
+++ b/safekeeper/client/src/mgmt_api.rs
@@ -8,8 +8,8 @@
use http_utils::error::HttpErrorBody;
use reqwest::{IntoUrl, Method, Response, StatusCode};
use safekeeper_api::models::{
- self, PullTimelineRequest, PullTimelineResponse, SafekeeperStatus, SafekeeperUtilization,
- TimelineCreateRequest,
+ self, OwnershipValidationRequest, OwnershipValidationResponse, PullTimelineRequest,
+ PullTimelineResponse, SafekeeperStatus, SafekeeperUtilization, TimelineCreateRequest,
};
use utils::id::{NodeId, TenantId, TimelineId};
use utils::logging::SecretString;
@@ -86,6 +86,15 @@ pub async fn create_timeline(&self, req: &TimelineCreateRequest) -> Result<reqwe
Ok(resp)
}
+ pub async fn validate_ownership(
+ &self,
+ req: &OwnershipValidationRequest,
+ ) -> Result<OwnershipValidationResponse> {
+ let uri = format!("{}/v1/ownership/validate", self.mgmt_api_endpoint);
+ let resp = self.post(&uri, req).await?;
+ resp.json().await.map_err(Error::ReceiveBody)
+ }
+
pub async fn pull_timeline(&self, req: &PullTimelineRequest) -> Result<PullTimelineResponse> {
let uri = format!("{}/v1/pull_timeline", self.mgmt_api_endpoint);
let resp = self.post(&uri, req).await?;
@@ -110,13 +119,24 @@ pub async fn delete_timeline(
&self,
tenant_id: TenantId,
timeline_id: TimelineId,
+ ownership_token: Option<&str>,
) -> Result<models::TimelineDeleteResult> {
let uri = format!(
"{}/v1/tenant/{}/timeline/{}",
self.mgmt_api_endpoint, tenant_id, timeline_id
);
- let resp = self
- .request_maybe_body(Method::DELETE, &uri, None::<()>)
+ let mut request = self.client.request(Method::DELETE, &uri);
+ if let Some(value) = &self.authorization_header {
+ request = request.header(reqwest::header::AUTHORIZATION, value.get_contents());
+ }
+ if let Some(token) = ownership_token {
+ request = request.header("Hakopod-Ownership-Token", token);
+ }
+ let resp = request
+ .send()
+ .await
+ .map_err(Error::ReceiveBody)?
+ .error_from_body()
.await?;
resp.json().await.map_err(Error::ReceiveBody)
}
diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs
index 6632819e1f5efbf9ed9f9d7551d05db1bcd6071e..ce4d06a7e4ab5014da8fabe44e34bf5db0778896 100644
--- a/safekeeper/src/bin/safekeeper.rs
+++ b/safekeeper/src/bin/safekeeper.rs
@@ -260,6 +260,9 @@ struct Args {
/// Run in development mode (disables security checks)
#[arg(long, help = "Run in development mode (disables security checks)")]
dev: bool,
+ /// Require durable Hakopod ownership records for timeline mutations.
+ #[arg(long, default_value = "false")]
+ hakopod_ownership_v1: bool,
/* BEGIN_HADRON */
#[arg(long)]
enable_pull_timeline_on_startup: bool,
@@ -295,6 +298,10 @@ async fn main() -> anyhow::Result<()> {
let mut matches = cmd.get_matches();
let mut args = <Args as clap::FromArgMatches>::from_arg_matches_mut(&mut matches)?;
+ if args.dev && args.hakopod_ownership_v1 {
+ anyhow::bail!("`--hakopod-ownership-v1` cannot be combined with `--dev`");
+ }
+
// I failed to modify opt_pathbuf_parser to return Option<PathBuf> in
// reasonable time, so turn empty string into option post factum.
if let Some(pb) = &args.pg_auth_public_key_path {
@@ -415,6 +422,7 @@ async fn main() -> anyhow::Result<()> {
};
let conf = Arc::new(SafeKeeperConf {
+ hakopod_ownership_v1: args.hakopod_ownership_v1,
workdir,
my_id: id,
listen_pg_addr: args.listen_pg,
diff --git a/safekeeper/src/hadron.rs b/safekeeper/src/hadron.rs
index 72b377fcc4370db92fedbdb47d9d61bdd971f86d..f5cfdf14534aa83d57f2a30d551cf84773d4eb34 100644
--- a/safekeeper/src/hadron.rs
+++ b/safekeeper/src/hadron.rs
@@ -190,6 +190,8 @@ pub async fn hcc_pull_timeline(
nodeid_http: &HashMap<u64, String>,
) -> bool {
let mut request = PullTimelineRequest {
+ ownership_token: None,
+ ownership_request_hash: None,
tenant_id: timeline.tenant_id,
timeline_id: timeline.timeline_id,
http_hosts: Vec::new(),
diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs
index 373589a18e780783578b7b1553b6b0ef80851910..1d96003f158d57305aff62caba042802d41c2c7a 100644
--- a/safekeeper/src/handler.rs
+++ b/safekeeper/src/handler.rs
@@ -335,6 +335,8 @@ fn process_query(
proto_version,
allow_timeline_creation,
} => {
+ let allow_timeline_creation =
+ allow_timeline_creation && !self.conf.hakopod_ownership_v1;
self.handle_start_wal_push(pgb, proto_version, allow_timeline_creation)
.instrument(info_span!("WAL receiver"))
.await
diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs
index 9f4c7141ec2fce4eed0602c09852dbe5ad80bd36..74fcb2d6c57039998e76300935bc9c60d50e61e9 100644
--- a/safekeeper/src/http/routes.rs
+++ b/safekeeper/src/http/routes.rs
@@ -17,11 +17,13 @@
use pem::Pem;
use postgres_ffi::WAL_SEGMENT_SIZE;
use safekeeper_api::models::{
- AcceptorStateStatus, PullTimelineRequest, SafekeeperStatus, SkTimelineInfo, TenantDeleteResult,
- TermSwitchApiEntry, TimelineCopyRequest, TimelineCreateRequest, TimelineDeleteResult,
- TimelineStatus, TimelineTermBumpRequest,
+ AcceptorStateStatus, OwnershipValidationRequest, OwnershipValidationResponse,
+ PullTimelineRequest, SafekeeperStatus, SkTimelineInfo, TenantDeleteResult, TermSwitchApiEntry,
+ TimelineCopyRequest, TimelineCreateRequest, TimelineDeleteResult, TimelineStatus,
+ TimelineTermBumpRequest,
};
use safekeeper_api::{ServerInfo, membership, models};
+use sha2::{Digest, Sha256};
use storage_broker::proto::{SafekeeperTimelineInfo, TenantTimelineId as ProtoTenantTimelineId};
use tokio::sync::mpsc;
use tokio::task;
@@ -49,6 +51,60 @@ async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiErr
json_response(StatusCode::OK, status)
}
+async fn ownership_validate_handler(
+ mut request: Request<Body>,
+) -> Result<Response<Body>, ApiError> {
+ const MAX_TIMELINES: usize = 4096;
+ check_permission(&request, None)?;
+ let mut data = json_request::<OwnershipValidationRequest>(&mut request).await?;
+ let conf = get_conf(&request);
+ if !conf.hakopod_ownership_v1 {
+ return Err(ApiError::Forbidden(
+ "ownership validation requires Hakopod ownership strict mode".to_string(),
+ ));
+ }
+ if data.timelines.len() > MAX_TIMELINES {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "ownership validation exceeds {MAX_TIMELINES} timelines"
+ )));
+ }
+ data.timelines
+ .sort_by_key(|entry| (entry.tenant_id, entry.timeline_id));
+ let mut hasher = Sha256::new();
+ hasher.update(b"hakopod-safekeeper-ownership-preflight-v1\0");
+ for entry in data.timelines {
+ let ttid = TenantTimelineId::new(entry.tenant_id, entry.timeline_id);
+ let record = crate::ownership::verify(
+ conf,
+ ttid,
+ &entry.ownership_token,
+ &entry.ownership_request_hash,
+ entry.allow_tombstoned,
+ )
+ .await
+ .map_err(|error| ApiError::Conflict(error.to_string()))?;
+ hasher.update(entry.tenant_id.to_string().as_bytes());
+ hasher.update(b"/");
+ hasher.update(entry.timeline_id.to_string().as_bytes());
+ hasher.update(b"\0");
+ hasher.update(record.token.as_bytes());
+ hasher.update(b"\0");
+ hasher.update(record.request_hash.as_bytes());
+ hasher.update(if record.tombstoned {
+ b"\0tombstoned\0".as_slice()
+ } else {
+ b"\0active\0".as_slice()
+ });
+ }
+ json_response(
+ StatusCode::OK,
+ OwnershipValidationResponse {
+ schema_version: 1,
+ digest: hex::encode(hasher.finalize()),
+ },
+ )
+}
+
fn get_conf(request: &Request<Body>) -> &SafeKeeperConf {
request
.data::<Arc<SafeKeeperConf>>()
@@ -75,6 +131,11 @@ async fn tenant_delete_handler(mut request: Request<Body>) -> Result<Response<Bo
let tenant_id = parse_request_param(&request, "tenant_id")?;
let only_local = parse_query_param(&request, "only_local")?.unwrap_or(false);
check_permission(&request, Some(tenant_id))?;
+ if get_conf(&request).hakopod_ownership_v1 {
+ return Err(ApiError::Forbidden(
+ "tenant-wide deletion is disabled by Hakopod ownership strict mode".to_string(),
+ ));
+ }
ensure_no_body(&mut request).await?;
let global_timelines = get_global_timelines(&request);
let action = if only_local {
@@ -102,12 +163,34 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
};
check_permission(&request, Some(ttid.tenant_id))?;
+ let conf = get_conf(&request);
+ let global_timelines = get_global_timelines(&request);
+ if conf.hakopod_ownership_v1 {
+ let token = request_data.ownership_token.as_deref().ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("missing timeline ownership token"))
+ })?;
+ let request_hash = request_data
+ .ownership_request_hash
+ .as_deref()
+ .ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("missing timeline ownership request hash"))
+ })?;
+ crate::ownership::reserve(
+ conf,
+ ttid,
+ token,
+ request_hash,
+ global_timelines.contains_timeline(&ttid),
+ )
+ .await
+ .map_err(|error| ApiError::Conflict(error.to_string()))?;
+ }
+
let server_info = ServerInfo {
pg_version: request_data.pg_version,
system_id: request_data.system_id.unwrap_or(0),
wal_seg_size: request_data.wal_seg_size.unwrap_or(WAL_SEGMENT_SIZE as u32),
};
- let global_timelines = get_global_timelines(&request);
global_timelines
.create(
ttid,
@@ -215,7 +298,34 @@ async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body
walsenders: tli.get_walsenders().get_all_public(),
walreceivers: tli.get_walreceivers().get_all(),
};
- json_response(StatusCode::OK, status)
+ if conf.hakopod_ownership_v1 {
+ let ownership = crate::ownership::describe(conf, ttid)
+ .await
+ .map_err(ApiError::InternalServerError)?
+ .ok_or_else(|| {
+ ApiError::Conflict("timeline has no durable ownership record".to_string())
+ })?;
+ let mut value = serde_json::to_value(status)
+ .map_err(|error| ApiError::InternalServerError(error.into()))?;
+ let object = value.as_object_mut().ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!("timeline status must be an object"))
+ })?;
+ object.insert(
+ "ownership_token".to_string(),
+ serde_json::Value::String(ownership.token),
+ );
+ object.insert(
+ "ownership_request_hash".to_string(),
+ serde_json::Value::String(ownership.request_hash),
+ );
+ object.insert(
+ "ownership_tombstoned".to_string(),
+ serde_json::Value::Bool(ownership.tombstoned),
+ );
+ json_response(StatusCode::OK, value)
+ } else {
+ json_response(StatusCode::OK, status)
+ }
}
/// Deactivates the timeline and removes its data directory.
@@ -226,6 +336,24 @@ async fn timeline_delete_handler(mut request: Request<Body>) -> Result<Response<
);
let only_local = parse_query_param(&request, "only_local")?.unwrap_or(false);
check_permission(&request, Some(ttid.tenant_id))?;
+ let conf = get_conf(&request);
+ if conf.hakopod_ownership_v1 {
+ if only_local {
+ return Err(ApiError::Forbidden(
+ "local-only deletion is disabled by Hakopod ownership strict mode".to_string(),
+ ));
+ }
+ let token = request
+ .headers()
+ .get("Hakopod-Ownership-Token")
+ .and_then(|value| value.to_str().ok())
+ .ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("missing timeline ownership token"))
+ })?;
+ crate::ownership::tombstone(conf, ttid, token)
+ .await
+ .map_err(|error| ApiError::Conflict(error.to_string()))?;
+ }
ensure_no_body(&mut request).await?;
let global_timelines = get_global_timelines(&request);
let action = if only_local {
@@ -247,7 +375,24 @@ async fn timeline_pull_handler(mut request: Request<Body>) -> Result<Response<Bo
let data: PullTimelineRequest = json_request(&mut request).await?;
let conf = get_conf(&request);
let global_timelines = get_global_timelines(&request);
-
+ if conf.hakopod_ownership_v1 {
+ let ttid = TenantTimelineId::new(data.tenant_id, data.timeline_id);
+ let token = data.ownership_token.as_deref().ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("missing timeline ownership token"))
+ })?;
+ let request_hash = data.ownership_request_hash.as_deref().ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("missing timeline ownership request hash"))
+ })?;
+ crate::ownership::reserve(
+ conf,
+ ttid,
+ token,
+ request_hash,
+ global_timelines.contains_timeline(&ttid),
+ )
+ .await
+ .map_err(|error| ApiError::Conflict(error.to_string()))?;
+ }
let ca_certs = conf
.ssl_ca_certs
.iter()
@@ -406,6 +551,11 @@ async fn timeline_membership_handler(
async fn timeline_copy_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
check_permission(&request, None)?;
+ if get_conf(&request).hakopod_ownership_v1 {
+ return Err(ApiError::Forbidden(
+ "timeline copy is disabled by Hakopod ownership strict mode".to_string(),
+ ));
+ }
let request_data: TimelineCopyRequest = json_request(&mut request).await?;
let source_ttid = TenantTimelineId::new(
@@ -739,6 +889,9 @@ pub fn make_router(
.get("/profile/cpu", |r| request_span(r, profile_cpu_handler))
.get("/profile/heap", |r| request_span(r, profile_heap_handler))
.get("/v1/status", |r| request_span(r, status_handler))
+ .post("/v1/ownership/validate", |r| {
+ request_span(r, ownership_validate_handler)
+ })
.put("/v1/failpoints", |r| {
request_span(r, move |r| async {
check_permission(&r, None)?;
diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs
index c6f9cc29e5a05c7b85c849a4d05787f5f55b9917..bf00930e88380dae90533cf1f45df9105bafa970 100644
--- a/safekeeper/src/lib.rs
+++ b/safekeeper/src/lib.rs
@@ -25,6 +25,7 @@
pub mod handler;
pub mod http;
pub mod metrics;
+pub mod ownership;
pub mod patch_control_file;
pub mod pull_timeline;
pub mod rate_limit;
@@ -92,6 +93,7 @@ pub mod defaults {
#[derive(Debug, Clone)]
pub struct SafeKeeperConf {
+ pub hakopod_ownership_v1: bool,
// Repository directory, relative to current working directory.
// Normally, the safekeeper changes the current working directory
// to the repository, and 'workdir' is always '.'. But we don't do
@@ -156,6 +158,7 @@ pub struct SafeKeeperConf {
impl SafeKeeperConf {
pub fn dummy() -> Self {
SafeKeeperConf {
+ hakopod_ownership_v1: false,
workdir: Utf8PathBuf::from("./"),
no_sync: false,
listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
diff --git a/safekeeper/src/ownership.rs b/safekeeper/src/ownership.rs
new file mode 100644
index 0000000000000000000000000000000000000000..7a53e191c184498e8dd431c53fcc3fe7b6e43680
--- /dev/null
+++ b/safekeeper/src/ownership.rs
@@ -0,0 +1,493 @@
+use std::fs::{File, OpenOptions};
+use std::io::{ErrorKind, Read, Write};
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result, bail};
+use camino::{Utf8Path, Utf8PathBuf};
+use nix::errno::Errno;
+use nix::fcntl::{Flock, FlockArg, OFlag};
+use serde::{Deserialize, Serialize};
+use tokio::fs;
+use utils::id::TenantTimelineId;
+
+#[cfg(unix)]
+use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
+
+use crate::SafeKeeperConf;
+
+const SCHEMA_VERSION: u8 = 1;
+const MAX_RECORD_BYTES: u64 = 4096;
+const LOCK_TIMEOUT: Duration = Duration::from_secs(5);
+const LOCK_RETRY: Duration = Duration::from_millis(10);
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+pub struct TimelineOwnership {
+ pub schema_version: u8,
+ pub token: String,
+ pub request_hash: String,
+ pub tombstoned: bool,
+}
+
+fn validate(token: &str, request_hash: &str) -> Result<()> {
+ if token.len() != 32
+ || token == "00000000000000000000000000000000"
+ || !token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ bail!("invalid Hakopod ownership token");
+ }
+ if request_hash.len() != 64
+ || !request_hash
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ bail!("invalid Hakopod ownership request hash");
+ }
+ Ok(())
+}
+
+fn directory(conf: &SafeKeeperConf) -> Utf8PathBuf {
+ conf.workdir.join("hakopod-ownership-v1")
+}
+
+fn record_path(conf: &SafeKeeperConf, ttid: TenantTimelineId) -> Utf8PathBuf {
+ directory(conf).join(format!("{}-{}.json", ttid.tenant_id, ttid.timeline_id))
+}
+
+fn tombstone_path(conf: &SafeKeeperConf, ttid: TenantTimelineId) -> Utf8PathBuf {
+ directory(conf).join(format!(
+ "{}-{}.tombstone.json",
+ ttid.tenant_id, ttid.timeline_id
+ ))
+}
+
+fn lock_path(conf: &SafeKeeperConf, ttid: TenantTimelineId) -> Utf8PathBuf {
+ directory(conf).join(format!("{}-{}.lock", ttid.tenant_id, ttid.timeline_id))
+}
+
+async fn ensure_directory(path: &Utf8Path) -> Result<()> {
+ fs::create_dir_all(path)
+ .await
+ .context("create ownership directory")?;
+ let metadata = fs::symlink_metadata(path)
+ .await
+ .context("inspect ownership directory")?;
+ if metadata.file_type().is_symlink() || !metadata.is_dir() {
+ bail!("ownership path must be a real directory");
+ }
+ #[cfg(unix)]
+ {
+ if metadata.uid() != nix::unistd::geteuid().as_raw() {
+ bail!("ownership directory must be owned by the safekeeper user");
+ }
+ fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))
+ .await
+ .context("secure ownership directory")?;
+ }
+ Ok(())
+}
+
+async fn read(path: &Utf8Path) -> Result<Option<TimelineOwnership>> {
+ let initial_metadata = match std::fs::symlink_metadata(path) {
+ Ok(metadata) => metadata,
+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
+ Err(error) => return Err(error).context("inspect ownership record"),
+ };
+ validate_private_file(&initial_metadata, "ownership record")?;
+ let mut options = OpenOptions::new();
+ options.read(true);
+ #[cfg(unix)]
+ options.custom_flags((OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK).bits());
+ let file = options.open(path).context("open ownership record")?;
+ let metadata = file.metadata().context("inspect opened ownership record")?;
+ validate_private_file(&metadata, "ownership record")?;
+ let path_metadata = std::fs::symlink_metadata(path).context("reinspect ownership record")?;
+ validate_private_file(&path_metadata, "ownership record")?;
+ if !same_file(&metadata, &path_metadata) || !same_file(&initial_metadata, &path_metadata) {
+ bail!("ownership record changed while opening");
+ }
+ let mut bytes = Vec::new();
+ file.take(MAX_RECORD_BYTES + 1)
+ .read_to_end(&mut bytes)
+ .context("read ownership record")?;
+ if bytes.len() as u64 > MAX_RECORD_BYTES {
+ bail!("ownership record exceeds its size bound");
+ }
+ let record: TimelineOwnership =
+ serde_json::from_slice(&bytes).context("decode ownership record")?;
+ if record.schema_version != SCHEMA_VERSION {
+ bail!("unsupported ownership record version");
+ }
+ validate(&record.token, &record.request_hash)?;
+ Ok(Some(record))
+}
+
+async fn create(path: &Utf8Path, record: &TimelineOwnership) -> Result<bool> {
+ let bytes = serde_json::to_vec(record).context("encode ownership record")?;
+ let parent = path.parent().context("ownership record has no parent")?;
+ let temporary = parent.join(format!(
+ ".hakopod-ownership-{:016x}.tmp",
+ rand::random::<u64>()
+ ));
+ let cleanup = scopeguard::guard(temporary.clone(), |path| {
+ let _ = std::fs::remove_file(path);
+ });
+ let mut options = OpenOptions::new();
+ options.write(true).create_new(true);
+ #[cfg(unix)]
+ {
+ options.mode(0o600).custom_flags(OFlag::O_NOFOLLOW.bits());
+ }
+ let mut file = match options.open(&temporary) {
+ Ok(file) => file,
+ Err(error) => return Err(error).context("create temporary ownership record"),
+ };
+ validate_private_file(
+ &file
+ .metadata()
+ .context("inspect temporary ownership record")?,
+ "temporary ownership record",
+ )?;
+ file.write_all(&bytes).context("write ownership record")?;
+ file.sync_all().context("sync ownership record")?;
+ drop(file);
+
+ let published = match std::fs::hard_link(&temporary, path) {
+ Ok(()) => true,
+ Err(error) if error.kind() == ErrorKind::AlreadyExists => false,
+ Err(error) => return Err(error).context("publish ownership record"),
+ };
+ drop(cleanup);
+ if published {
+ std::fs::File::open(parent)
+ .and_then(|directory| directory.sync_all())
+ .context("sync ownership directory")?;
+ }
+ Ok(published)
+}
+
+async fn acquire_resource_lock(
+ conf: &SafeKeeperConf,
+ ttid: TenantTimelineId,
+) -> Result<Flock<File>> {
+ let directory = directory(conf);
+ ensure_directory(&directory).await?;
+ let path = lock_path(conf, ttid);
+ let deadline = Instant::now() + LOCK_TIMEOUT;
+ loop {
+ let mut options = OpenOptions::new();
+ options.read(true).write(true).create(true);
+ #[cfg(unix)]
+ options
+ .mode(0o600)
+ .custom_flags((OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK).bits());
+ let file = options.open(&path).context("open ownership lock")?;
+ let metadata = file.metadata().context("inspect ownership lock")?;
+ validate_private_file(&metadata, "ownership lock")?;
+ match Flock::lock(file, FlockArg::LockExclusiveNonblock) {
+ Ok(lock) => {
+ let path_metadata =
+ std::fs::symlink_metadata(&path).context("reinspect ownership lock")?;
+ validate_private_file(&path_metadata, "ownership lock")?;
+ if same_file(&metadata, &path_metadata) {
+ return Ok(lock);
+ }
+ }
+ Err((_, Errno::EAGAIN)) => {}
+ Err((_, error)) => return Err(error).context("lock ownership state"),
+ }
+ if Instant::now() >= deadline {
+ bail!("timed out locking ownership state");
+ }
+ tokio::time::sleep(LOCK_RETRY).await;
+ }
+}
+
+fn validate_private_file(metadata: &std::fs::Metadata, description: &str) -> Result<()> {
+ if !metadata.is_file() || metadata.len() > MAX_RECORD_BYTES {
+ bail!("{description} is unsafe or exceeds its size bound");
+ }
+ #[cfg(unix)]
+ {
+ if metadata.uid() != nix::unistd::geteuid().as_raw() {
+ bail!("{description} must be owned by the safekeeper user");
+ }
+ if metadata.permissions().mode() & 0o777 != 0o600 {
+ bail!("{description} must have mode 0600");
+ }
+ }
+ Ok(())
+}
+
+#[cfg(unix)]
+fn same_file(left: &std::fs::Metadata, right: &std::fs::Metadata) -> bool {
+ left.dev() == right.dev() && left.ino() == right.ino()
+}
+
+#[cfg(not(unix))]
+fn same_file(_left: &std::fs::Metadata, _right: &std::fs::Metadata) -> bool {
+ true
+}
+
+pub async fn reserve(
+ conf: &SafeKeeperConf,
+ ttid: TenantTimelineId,
+ token: &str,
+ request_hash: &str,
+ resource_exists: bool,
+) -> Result<()> {
+ validate(token, request_hash)?;
+ let _lock = acquire_resource_lock(conf, ttid).await?;
+ if read(&tombstone_path(conf, ttid)).await?.is_some() {
+ bail!("timeline ownership is permanently tombstoned");
+ }
+ let wanted = TimelineOwnership {
+ schema_version: SCHEMA_VERSION,
+ token: token.to_owned(),
+ request_hash: request_hash.to_owned(),
+ tombstoned: false,
+ };
+ let path = record_path(conf, ttid);
+ if let Some(stored) = read(&path).await? {
+ if stored != wanted {
+ bail!("timeline ownership token or request hash conflicts");
+ }
+ } else {
+ if resource_exists {
+ bail!("refusing to adopt a pre-existing unowned timeline");
+ }
+ if !create(&path, &wanted).await? && read(&path).await?.as_ref() != Some(&wanted) {
+ bail!("timeline ownership token or request hash conflicts");
+ }
+ }
+ if read(&tombstone_path(conf, ttid)).await?.is_some() {
+ bail!("timeline ownership is permanently tombstoned");
+ }
+ Ok(())
+}
+
+pub async fn describe(
+ conf: &SafeKeeperConf,
+ ttid: TenantTimelineId,
+) -> Result<Option<TimelineOwnership>> {
+ let _lock = acquire_resource_lock(conf, ttid).await?;
+ if let Some(tombstone) = read(&tombstone_path(conf, ttid)).await? {
+ return Ok(Some(tombstone));
+ }
+ read(&record_path(conf, ttid)).await
+}
+
+pub async fn verify(
+ conf: &SafeKeeperConf,
+ ttid: TenantTimelineId,
+ token: &str,
+ request_hash: &str,
+ allow_tombstoned: bool,
+) -> Result<TimelineOwnership> {
+ validate(token, request_hash)?;
+ let record = describe(conf, ttid)
+ .await?
+ .context("timeline has no durable ownership record")?;
+ if record.token != token || record.request_hash != request_hash {
+ bail!("timeline ownership token or request hash conflicts");
+ }
+ if record.tombstoned && !allow_tombstoned {
+ bail!("timeline ownership is permanently tombstoned");
+ }
+ Ok(record)
+}
+
+pub async fn tombstone(conf: &SafeKeeperConf, ttid: TenantTimelineId, token: &str) -> Result<()> {
+ let _lock = acquire_resource_lock(conf, ttid).await?;
+ let record = read(&record_path(conf, ttid))
+ .await?
+ .context("timeline has no durable ownership record")?;
+ if record.token != token {
+ bail!("timeline deletion ownership token does not match");
+ }
+ let tombstone = TimelineOwnership {
+ tombstoned: true,
+ ..record
+ };
+ let path = tombstone_path(conf, ttid);
+ if !create(&path, &tombstone).await? && read(&path).await?.as_ref() != Some(&tombstone) {
+ bail!("timeline ownership tombstone conflicts");
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::str::FromStr;
+ use utils::id::{TenantId, TimelineId};
+
+ #[cfg(unix)]
+ use std::os::unix::fs::{PermissionsExt, symlink};
+
+ fn id(value: &str) -> TenantTimelineId {
+ TenantTimelineId::new(
+ TenantId::from_str(value).unwrap(),
+ TimelineId::from_str("22222222222222222222222222222222").unwrap(),
+ )
+ }
+
+ #[tokio::test]
+ async fn replay_conflict_restart_and_tombstone_are_durable() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("11111111111111111111111111111111");
+ let token = "33333333333333333333333333333333";
+ let hash = "4444444444444444444444444444444444444444444444444444444444444444";
+ reserve(&conf, ttid, token, hash, false).await.unwrap();
+ reserve(&conf, ttid, token, hash, true).await.unwrap();
+ assert!(
+ reserve(&conf, ttid, "55555555555555555555555555555555", hash, true)
+ .await
+ .is_err()
+ );
+ tombstone(&conf, ttid, token).await.unwrap();
+ assert!(describe(&conf, ttid).await.unwrap().unwrap().tombstoned);
+ assert!(reserve(&conf, ttid, token, hash, false).await.is_err());
+ }
+
+ #[tokio::test]
+ async fn concurrent_replay_never_observes_a_partial_record() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("66666666666666666666666666666666");
+ let token = "77777777777777777777777777777777";
+ let hash = "8888888888888888888888888888888888888888888888888888888888888888";
+
+ let (left, right) = tokio::join!(
+ reserve(&conf, ttid, token, hash, false),
+ reserve(&conf, ttid, token, hash, false),
+ );
+ left.unwrap();
+ right.unwrap();
+ assert_eq!(
+ describe(&conf, ttid).await.unwrap(),
+ Some(TimelineOwnership {
+ schema_version: SCHEMA_VERSION,
+ token: token.to_owned(),
+ request_hash: hash.to_owned(),
+ tombstoned: false,
+ })
+ );
+ }
+
+ #[tokio::test]
+ async fn refuses_to_adopt_pre_existing_timeline() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("99999999999999999999999999999999");
+ assert!(
+ reserve(
+ &conf,
+ ttid,
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+ true,
+ )
+ .await
+ .is_err()
+ );
+ assert!(describe(&conf, ttid).await.unwrap().is_none());
+ }
+
+ #[tokio::test]
+ async fn verify_is_read_only_and_fails_closed() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+ let token = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ let hash = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
+
+ assert!(verify(&conf, ttid, token, hash, false).await.is_err());
+ reserve(&conf, ttid, token, hash, false).await.unwrap();
+ let before = std::fs::read(record_path(&conf, ttid)).unwrap();
+ assert_eq!(
+ verify(&conf, ttid, token, hash, false).await.unwrap().token,
+ token
+ );
+ assert!(
+ verify(&conf, ttid, "dddddddddddddddddddddddddddddddd", hash, false,)
+ .await
+ .is_err()
+ );
+ assert_eq!(std::fs::read(record_path(&conf, ttid)).unwrap(), before);
+
+ tombstone(&conf, ttid, token).await.unwrap();
+ assert!(verify(&conf, ttid, token, hash, false).await.is_err());
+ assert!(
+ verify(&conf, ttid, token, hash, true)
+ .await
+ .unwrap()
+ .tombstoned
+ );
+ }
+
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn rejects_symlinked_and_permissive_ownership_records() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+ let token = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ let hash = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
+
+ ensure_directory(&directory(&conf)).await.unwrap();
+ let target = directory(&conf).join("attacker-controlled.json");
+ std::fs::write(&target, b"{}").unwrap();
+ symlink(&target, record_path(&conf, ttid)).unwrap();
+ assert!(reserve(&conf, ttid, token, hash, false).await.is_err());
+
+ std::fs::remove_file(record_path(&conf, ttid)).unwrap();
+ let record = TimelineOwnership {
+ schema_version: SCHEMA_VERSION,
+ token: token.to_owned(),
+ request_hash: hash.to_owned(),
+ tombstoned: false,
+ };
+ std::fs::write(
+ record_path(&conf, ttid),
+ serde_json::to_vec(&record).unwrap(),
+ )
+ .unwrap();
+ std::fs::set_permissions(
+ record_path(&conf, ttid),
+ std::fs::Permissions::from_mode(0o644),
+ )
+ .unwrap();
+ assert!(describe(&conf, ttid).await.is_err());
+ }
+
+ #[tokio::test]
+ async fn concurrent_reserve_and_delete_leave_a_permanent_tombstone() {
+ let dir = camino_tempfile::tempdir().unwrap();
+ let mut conf = SafeKeeperConf::dummy();
+ conf.workdir = dir.path().to_owned();
+ let ttid = id("dddddddddddddddddddddddddddddddd");
+ let token = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee";
+ let hash = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff";
+ reserve(&conf, ttid, token, hash, false).await.unwrap();
+
+ let (replay, deletion) = tokio::join!(
+ reserve(&conf, ttid, token, hash, true),
+ tombstone(&conf, ttid, token),
+ );
+ deletion.unwrap();
+ if replay.is_ok() {
+ assert!(reserve(&conf, ttid, token, hash, true).await.is_err());
+ }
+ let described = describe(&conf, ttid).await.unwrap().unwrap();
+ assert!(described.tombstoned);
+ assert_eq!(described.token, token);
+ }
+}
diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs
index f63d1abdcf0e3055bbfa69ff8ccf94f855ad6c8e..2621df6b5597b8133c452c6f91d4763312c35c70 100644
--- a/safekeeper/src/timelines_global_map.rs
+++ b/safekeeper/src/timelines_global_map.rs
@@ -271,6 +271,11 @@ pub fn timelines_count(&self) -> usize {
self.state.lock().unwrap().timelines.len()
}
+ /// Return whether a timeline is loaded or is currently being created.
+ pub(crate) fn contains_timeline(&self, ttid: &TenantTimelineId) -> bool {
+ self.state.lock().unwrap().timelines.contains_key(ttid)
+ }
+
/// Get the global safekeeper config.
pub fn get_global_config(&self) -> Arc<SafeKeeperConf> {
self.state.lock().unwrap().conf.clone()
diff --git a/safekeeper/tests/walproposer_sim/safekeeper.rs b/safekeeper/tests/walproposer_sim/safekeeper.rs
index 30d3ab1a872c2d0dac58e98af013da2efb6b0db1..adee94440664ef0a9b5cb3871d2195ed7a718a80 100644
--- a/safekeeper/tests/walproposer_sim/safekeeper.rs
+++ b/safekeeper/tests/walproposer_sim/safekeeper.rs
@@ -148,6 +148,7 @@ pub fn run_server(os: NodeOs, disk: Arc<SafekeeperDisk>) -> Result<()> {
debug!("started server");
os.log_event("started;safekeeper".to_owned());
let conf = SafeKeeperConf {
+ hakopod_ownership_v1: false,
workdir: Utf8PathBuf::from("."),
my_id: NodeId(os.id() as u64),
listen_pg_addr: String::new(),
diff --git a/storage_controller/Cargo.toml b/storage_controller/Cargo.toml
index d67be6d469e67e29291187dbe9d6d721d05b65c3..373cbee745fd7e90cd9c07fa63ed252a9e2d9b14 100644
--- a/storage_controller/Cargo.toml
+++ b/storage_controller/Cargo.toml
@@ -47,6 +47,7 @@ regex.workspace = true
rustls-native-certs.workspace = true
serde.workspace = true
serde_json.workspace = true
+sha2.workspace = true
thiserror.workspace = true
tokio-rustls.workspace = true
tokio-util.workspace = true
diff --git a/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/down.sql b/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/down.sql
new file mode 100644
index 0000000000000000000000000000000000000000..6be6004118e2dd71e2e65fa052e1e70b63c3723d
--- /dev/null
+++ b/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/down.sql
@@ -0,0 +1,6 @@
+ALTER TABLE timelines
+ DROP CONSTRAINT timeline_ownership_pair,
+ DROP COLUMN ownership_request_hash,
+ DROP COLUMN ownership_token;
+DROP TABLE hakopod_resource_ownership;
+DROP TYPE hakopod_resource_ownership_state;
diff --git a/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/up.sql b/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/up.sql
new file mode 100644
index 0000000000000000000000000000000000000000..ef745dc95fd60ead7fc248367368283aa706bf90
--- /dev/null
+++ b/storage_controller/migrations/2026-10-01-000001_hakopod_resource_ownership/up.sql
@@ -0,0 +1,28 @@
+CREATE TYPE hakopod_resource_ownership_state AS ENUM
+ ('reserved', 'applying', 'completed', 'failed', 'deleting', 'tombstoned');
+
+CREATE TABLE hakopod_resource_ownership (
+ ownership_token UUID PRIMARY KEY,
+ resource_kind TEXT NOT NULL,
+ external_key TEXT NOT NULL,
+ request_hash BYTEA NOT NULL CHECK (octet_length(request_hash) = 32),
+ state hakopod_resource_ownership_state NOT NULL,
+ result JSONB,
+ delete_token UUID,
+ error_code TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ UNIQUE (resource_kind, external_key),
+ CHECK (
+ (state IN ('completed', 'deleting', 'tombstoned')) = (result IS NOT NULL)
+ ),
+ CHECK (state <> 'failed' OR error_code IS NOT NULL),
+ CHECK (delete_token IS NULL OR state IN ('deleting', 'tombstoned'))
+);
+
+ALTER TABLE timelines
+ ADD COLUMN ownership_token TEXT,
+ ADD COLUMN ownership_request_hash TEXT,
+ ADD CONSTRAINT timeline_ownership_pair CHECK (
+ (ownership_token IS NULL) = (ownership_request_hash IS NULL)
+ );
diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs
index b40da4fd655a86f56e204c1ac6b7d03ebdf06116..80c7be3f954628562106bcc8cca4c9361bf2e0c1 100644
--- a/storage_controller/src/http.rs
+++ b/storage_controller/src/http.rs
@@ -14,6 +14,7 @@
use http_utils::json::{json_request, json_response};
use http_utils::request::{must_get_query_param, parse_query_param, parse_request_param};
use http_utils::{RequestExt, RouterBuilder};
+use hyper::body::HttpBody as _;
use hyper::header::CONTENT_TYPE;
use hyper::{Body, Request, Response, StatusCode, Uri};
use metrics::{BuildInfo, NeonMetrics};
@@ -34,6 +35,9 @@
PutTimelineImportStatusRequest, ReAttachRequest, TimelineImportStatusRequest, ValidateRequest,
};
use pageserver_client::{BlockUnblock, mgmt_api};
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
use routerify::Middleware;
use tokio_util::sync::CancellationToken;
@@ -46,6 +50,7 @@
HttpRequestLatencyLabelGroup, HttpRequestStatusLabelGroup, METRICS_REGISTRY,
PageserverRequestLabelGroup,
};
+use crate::persistence::OwnershipDecision;
use crate::persistence::SafekeeperUpsert;
use crate::reconciler::ReconcileError;
use crate::service::{
@@ -53,6 +58,9 @@
TenantMutationLocations,
};
+const MAX_OWNED_REQUEST_BODY_BYTES: usize = 1024 * 1024;
+const OWNED_REQUEST_BODY_TIMEOUT: Duration = Duration::from_secs(10);
+
/// State available to HTTP request handlers
pub struct HttpState {
service: Arc<crate::service::Service>,
@@ -94,6 +102,117 @@ fn get_state(request: &Request<Body>) -> &HttpState {
.as_ref()
}
+async fn owned_json_request<T: DeserializeOwned + Serialize, F: FnOnce(&T) -> String>(
+ request: &mut Request<Body>,
+ resource_kind: &str,
+ external_key: F,
+) -> Result<(T, crate::ownership::OwnershipRequest), ApiError> {
+ // Reject missing credentials before accepting or buffering an attacker
+ // controlled body. The ownership parser validates the same token again
+ // after normalization so this early check cannot become an auth bypass.
+ crate::ownership::token(request)?;
+ let bytes = read_owned_request_body(
+ request.body_mut(),
+ MAX_OWNED_REQUEST_BODY_BYTES,
+ OWNED_REQUEST_BODY_TIMEOUT,
+ )
+ .await?;
+ if bytes.is_empty() {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "missing request body"
+ )));
+ }
+ let value: Value = serde_json::from_slice(&bytes)
+ .context("Failed to parse owned request JSON")
+ .map_err(ApiError::BadRequest)?;
+ let parsed = serde_json::from_value(value)
+ .context("Failed to parse owned request")
+ .map_err(ApiError::BadRequest)?;
+ let external_key = external_key(&parsed);
+ // Hash the typed request model so omitted defaults, explicit nulls, and
+ // ignored unknown fields cannot produce different hashes for the same
+ // provider mutation.
+ let value = serde_json::to_value(&parsed)
+ .context("Failed to normalize owned request")
+ .map_err(ApiError::BadRequest)?;
+ let ownership = crate::ownership::parse(request, resource_kind, &external_key, value)?;
+ Ok((parsed, ownership))
+}
+
+async fn read_owned_request_body(
+ body: &mut Body,
+ max_bytes: usize,
+ timeout: Duration,
+) -> Result<Vec<u8>, ApiError> {
+ tokio::time::timeout(timeout, async {
+ let mut bytes = Vec::new();
+ while let Some(chunk) = body.data().await {
+ let chunk = chunk
+ .context("Failed to read owned request body")
+ .map_err(ApiError::BadRequest)?;
+ let next_len = bytes.len().checked_add(chunk.len()).ok_or_else(|| {
+ ApiError::BadRequest(anyhow::anyhow!("owned request body is too large"))
+ })?;
+ if next_len > max_bytes {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "owned request body exceeds {max_bytes} bytes"
+ )));
+ }
+ bytes.extend_from_slice(&chunk);
+ }
+ Ok(bytes)
+ })
+ .await
+ .map_err(|_| ApiError::Timeout("owned request body timed out".into()))?
+}
+
+fn ownership_result<T: Serialize>(value: T, token: uuid::Uuid) -> Result<Value, ApiError> {
+ let mut value =
+ serde_json::to_value(value).map_err(|e| ApiError::InternalServerError(e.into()))?;
+ let object = value.as_object_mut().ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!("owned mutation result must be an object"))
+ })?;
+ object.insert(
+ "ownership_token".to_string(),
+ Value::String(token.simple().to_string()),
+ );
+ Ok(value)
+}
+
+async fn add_describe_ownership<T: Serialize>(
+ service: &Service,
+ resource_kind: &str,
+ external_key: &str,
+ value: T,
+) -> Result<Value, ApiError> {
+ let mut value =
+ serde_json::to_value(value).map_err(|e| ApiError::InternalServerError(e.into()))?;
+ if let Some(record) = service.get_ownership(resource_kind, external_key).await? {
+ if matches!(record.state.as_str(), "completed" | "deleting") {
+ let object = value.as_object_mut().ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!(
+ "owned describe result must be an object"
+ ))
+ })?;
+ object.insert(
+ "ownership_token".to_string(),
+ Value::String(record.token.simple().to_string()),
+ );
+ object.insert(
+ "ownership_state".to_string(),
+ Value::String(record.state.clone()),
+ );
+ if let Some(delete_token) = record.delete_token {
+ object.insert(
+ "ownership_delete_token".to_string(),
+ Value::String(delete_token.simple().to_string()),
+ );
+ }
+ }
+ }
+ Ok(value)
+}
+
/// Rate limits tenant requests.
///
/// TODO: this should be a request middleware, but requires us to extract the tenant ID from
@@ -139,8 +258,24 @@ async fn handle_re_attach(req: Request<Body>) -> Result<Response<Body>, ApiError
ForwardOutcome::NotForwarded(req) => req,
};
- let reattach_req = json_request::<ReAttachRequest>(&mut req).await?;
+ let mut reattach_req = json_request::<ReAttachRequest>(&mut req).await?;
let state = get_state(&req);
+
+ if state.service.get_config().hakopod_ownership_v1 && reattach_req.register.is_some() {
+ let ownership = state
+ .service
+ .get_ownership("pageserver_registration", &reattach_req.node_id.to_string())
+ .await?;
+ if !ownership.is_some_and(|record| record.state == "completed") {
+ return Err(ApiError::Forbidden(
+ "pageserver re-attach cannot create an unowned registration".to_string(),
+ ));
+ }
+ // Registration is immutable in ownership mode. The dedicated owned
+ // registration route has already persisted the exact node identity;
+ // re-attach may only refresh generations for that registered node.
+ reattach_req.register = None;
+ }
json_response(StatusCode::OK, state.service.re_attach(reattach_req).await?)
}
@@ -262,12 +397,42 @@ async fn handle_tenant_create(
ForwardOutcome::NotForwarded(req) => req,
};
- let create_req = json_request::<TenantCreateRequest>(&mut req).await?;
+ if !service.get_config().hakopod_ownership_v1 {
+ let create_req = json_request::<TenantCreateRequest>(&mut req).await?;
+ return json_response(
+ StatusCode::CREATED,
+ service.tenant_create(create_req).await?,
+ );
+ }
- json_response(
- StatusCode::CREATED,
- service.tenant_create(create_req).await?,
- )
+ let (create_req, ownership) =
+ owned_json_request::<TenantCreateRequest, _>(&mut req, "tenant", |body| {
+ body.new_tenant_id.tenant_id.to_string()
+ })
+ .await?;
+ let external_key = create_req.new_tenant_id.tenant_id.to_string();
+ match service
+ .reserve_ownership(
+ ownership.token,
+ "tenant",
+ &external_key,
+ &ownership.request_hash,
+ )
+ .await?
+ {
+ OwnershipDecision::Replay(result) => json_response(StatusCode::CREATED, result),
+ OwnershipDecision::Gone => Err(ApiError::Conflict(
+ "tenant ownership is permanently tombstoned".to_string(),
+ )),
+ OwnershipDecision::Begin | OwnershipDecision::Resume => {
+ let result =
+ ownership_result(service.tenant_create(create_req).await?, ownership.token)?;
+ service
+ .complete_ownership(ownership.token, result.clone())
+ .await?;
+ json_response(StatusCode::CREATED, result)
+ }
+ }
}
async fn handle_tenant_location_config(
@@ -422,22 +587,55 @@ async fn handle_tenant_delete(
req: Request<Body>,
) -> Result<Response<Body>, ApiError> {
let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?;
+ let validate_only = parse_query_param(&req, "validate_only")?.unwrap_or(false);
+ let prepare_delete = parse_query_param(&req, "prepare_delete")?.unwrap_or(false);
+ if validate_only && prepare_delete {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "tenant deletion cannot validate and prepare in one request"
+ )));
+ }
check_permissions(&req, Scope::PageServerApi)?;
maybe_rate_limit(&req, tenant_id).await;
- match maybe_forward(req).await {
+ let req = match maybe_forward(req).await {
ForwardOutcome::Forwarded(res) => {
return res;
}
- ForwardOutcome::NotForwarded(_req) => {}
+ ForwardOutcome::NotForwarded(req) => req,
};
- let status_code = service
- .tenant_delete(tenant_id)
- .await
- .and_then(map_reqwest_hyper_status)?;
+ let owned_token = if service.get_config().hakopod_ownership_v1 {
+ Some(crate::ownership::token(&req)?)
+ } else {
+ if validate_only || prepare_delete {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "ownership preflight requires Hakopod ownership strict mode"
+ )));
+ }
+ None
+ };
+
+ let delete_token =
+ if service.get_config().hakopod_ownership_v1 && !validate_only && !prepare_delete {
+ Some(crate::ownership::delete_token(&req)?)
+ } else {
+ None
+ };
+
+ let (status_code, response) = service
+ .tenant_delete(
+ tenant_id,
+ owned_token,
+ validate_only,
+ prepare_delete,
+ delete_token,
+ )
+ .await?;
+ let status_code = map_reqwest_hyper_status(status_code)?;
- if status_code == StatusCode::NOT_FOUND {
+ if let Some(response) = response {
+ json_response(status_code, response)
+ } else if status_code == StatusCode::NOT_FOUND {
// The pageserver uses 404 for successful deletion, but we use 200
json_response(StatusCode::OK, ())
} else {
@@ -460,13 +658,34 @@ async fn handle_tenant_timeline_create(
ForwardOutcome::NotForwarded(req) => req,
};
- let create_req = json_request::<TimelineCreateRequest>(&mut req).await?;
- json_response(
- StatusCode::CREATED,
- service
- .tenant_timeline_create(tenant_id, create_req)
- .await?,
- )
+ if !service.get_config().hakopod_ownership_v1 {
+ let create_req = json_request::<TimelineCreateRequest>(&mut req).await?;
+ return json_response(
+ StatusCode::CREATED,
+ service
+ .tenant_timeline_create(tenant_id, create_req)
+ .await?,
+ );
+ }
+ let (create_req, ownership) =
+ owned_json_request::<TimelineCreateRequest, _>(&mut req, "timeline", |body| {
+ format!("{tenant_id}/{}", body.new_timeline_id)
+ })
+ .await?;
+ if create_req.ownership_token.is_some() || create_req.ownership_request_hash.is_some() {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "timeline ownership metadata is supplied by the storage controller"
+ )));
+ }
+ let result = service
+ .tenant_timeline_create_owned(
+ tenant_id,
+ create_req,
+ ownership.token,
+ ownership.request_hash,
+ )
+ .await?;
+ json_response(StatusCode::CREATED, result)
}
async fn handle_tenant_timeline_delete(
@@ -479,11 +698,31 @@ async fn handle_tenant_timeline_delete(
check_permissions(&req, Scope::PageServerApi)?;
maybe_rate_limit(&req, tenant_id).await;
- match maybe_forward(req).await {
+ let req = match maybe_forward(req).await {
ForwardOutcome::Forwarded(res) => {
return res;
}
- ForwardOutcome::NotForwarded(_req) => {}
+ ForwardOutcome::NotForwarded(req) => req,
+ };
+
+ let owned_token = if service.get_config().hakopod_ownership_v1 {
+ let token = crate::ownership::token(&req)?;
+ match service
+ .begin_ownership_delete(token, "timeline", &format!("{tenant_id}/{timeline_id}"))
+ .await?
+ {
+ OwnershipDecision::Gone => return json_response(StatusCode::OK, ()),
+ OwnershipDecision::Begin | OwnershipDecision::Resume => {
+ Some(token.simple().to_string())
+ }
+ OwnershipDecision::Replay(_) => {
+ return Err(ApiError::Conflict(
+ "timeline is not in a deletable ownership state".to_string(),
+ ));
+ }
+ }
+ } else {
+ None
};
service
@@ -541,13 +780,21 @@ async fn deletion_wrapper<R, F>(service: Arc<Service>, f: F) -> Result<Response<
}
}
- deletion_wrapper(service, move |service| async move {
- service
- .tenant_timeline_delete(tenant_id, timeline_id)
- .await
- .and_then(map_reqwest_hyper_status)
+ let result = deletion_wrapper(service.clone(), move |service| {
+ let owned_token = owned_token.clone();
+ async move {
+ service
+ .tenant_timeline_delete(tenant_id, timeline_id, owned_token)
+ .await
+ .and_then(map_reqwest_hyper_status)
+ }
})
- .await
+ .await?;
+ if service.get_config().hakopod_ownership_v1 && result.status() == StatusCode::OK {
+ let token = crate::ownership::token(&req)?;
+ service.finish_ownership_delete(token).await?;
+ }
+ Ok(result)
}
async fn handle_tenant_timeline_archival_config(
@@ -889,7 +1136,15 @@ async fn handle_tenant_describe(
ForwardOutcome::NotForwarded(_req) => {}
};
- json_response(StatusCode::OK, service.tenant_describe(tenant_id)?)
+ let described = service.tenant_describe(tenant_id)?;
+ if service.get_config().hakopod_ownership_v1 {
+ json_response(
+ StatusCode::OK,
+ add_describe_ownership(&service, "tenant", &tenant_id.to_string(), described).await?,
+ )
+ } else {
+ json_response(StatusCode::OK, described)
+ }
}
/* BEGIN_HADRON */
@@ -908,12 +1163,23 @@ async fn handle_tenant_timeline_describe(
ForwardOutcome::NotForwarded(_req) => {}
};
- json_response(
- StatusCode::OK,
- service
- .tenant_timeline_describe(tenant_id, timeline_id)
+ let described = service
+ .tenant_timeline_describe(tenant_id, timeline_id)
+ .await?;
+ if service.get_config().hakopod_ownership_v1 {
+ json_response(
+ StatusCode::OK,
+ add_describe_ownership(
+ &service,
+ "timeline",
+ &format!("{tenant_id}/{timeline_id}"),
+ described,
+ )
.await?,
- )
+ )
+ } else {
+ json_response(StatusCode::OK, described)
+ }
}
/* END_HADRON */
@@ -947,10 +1213,50 @@ async fn handle_node_register(req: Request<Body>) -> Result<Response<Body>, ApiE
ForwardOutcome::NotForwarded(req) => req,
};
- let register_req = json_request::<NodeRegisterRequest>(&mut req).await?;
- let state = get_state(&req);
- state.service.node_register(register_req).await?;
- json_response(StatusCode::OK, ())
+ let service = get_state(&req).service.clone();
+ if !service.get_config().hakopod_ownership_v1 {
+ let register_req = json_request::<NodeRegisterRequest>(&mut req).await?;
+ service.node_register(register_req).await?;
+ return json_response(StatusCode::OK, ());
+ }
+ let (register_req, ownership) =
+ owned_json_request::<NodeRegisterRequest, _>(&mut req, "pageserver_registration", |body| {
+ body.node_id.to_string()
+ })
+ .await?;
+ let node_id = register_req.node_id;
+ match service
+ .reserve_ownership(
+ ownership.token,
+ "pageserver_registration",
+ &node_id.to_string(),
+ &ownership.request_hash,
+ )
+ .await?
+ {
+ OwnershipDecision::Replay(result) => json_response(StatusCode::OK, result),
+ OwnershipDecision::Gone => Err(ApiError::Conflict(
+ "pageserver registration is permanently tombstoned".to_string(),
+ )),
+ OwnershipDecision::Begin | OwnershipDecision::Resume => {
+ service.node_register(register_req).await?;
+ let node = service
+ .node_list()
+ .await?
+ .into_iter()
+ .find(|node| node.get_id() == node_id)
+ .ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!(
+ "registered pageserver disappeared"
+ ))
+ })?;
+ let result = ownership_result(node.describe(), ownership.token)?;
+ service
+ .complete_ownership(ownership.token, result.clone())
+ .await?;
+ json_response(StatusCode::OK, result)
+ }
+ }
}
async fn handle_node_list(req: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -1082,12 +1388,24 @@ async fn handle_node_status(req: Request<Body>) -> Result<Response<Body>, ApiErr
ForwardOutcome::NotForwarded(req) => req,
};
- let state = get_state(&req);
+ let service = get_state(&req).service.clone();
let node_id: NodeId = parse_request_param(&req, "node_id")?;
- let node_status = state.service.get_node(node_id).await?.describe();
-
- json_response(StatusCode::OK, node_status)
+ let node_status = service.get_node(node_id).await?.describe();
+ if service.get_config().hakopod_ownership_v1 {
+ json_response(
+ StatusCode::OK,
+ add_describe_ownership(
+ &service,
+ "pageserver_registration",
+ &node_id.to_string(),
+ node_status,
+ )
+ .await?,
+ )
+ } else {
+ json_response(StatusCode::OK, node_status)
+ }
}
async fn handle_node_shards(req: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -1692,11 +2010,14 @@ async fn handle_get_safekeeper(req: Request<Body>) -> Result<Response<Body>, Api
ForwardOutcome::NotForwarded(req) => req,
};
- let state = get_state(&req);
-
- let res = state.service.get_safekeeper(id).await;
+ let service = get_state(&req).service.clone();
+ let res = service.get_safekeeper(id).await;
match res {
+ Ok(b) if service.get_config().hakopod_ownership_v1 => json_response(
+ StatusCode::OK,
+ add_describe_ownership(&service, "safekeeper_registration", &id.to_string(), b).await?,
+ ),
Ok(b) => json_response(StatusCode::OK, b),
Err(crate::persistence::DatabaseError::Query(diesel::result::Error::NotFound)) => {
Err(ApiError::NotFound("unknown instance id".into()))
@@ -1709,11 +2030,30 @@ async fn handle_get_safekeeper(req: Request<Body>) -> Result<Response<Body>, Api
///
/// Assumes information is only relayed to storage controller after first selecting an unique id on
/// control plane database, which means we have an id field in the request and payload.
-async fn handle_upsert_safekeeper(mut req: Request<Body>) -> Result<Response<Body>, ApiError> {
+async fn handle_upsert_safekeeper(req: Request<Body>) -> Result<Response<Body>, ApiError> {
check_permissions(&req, Scope::Infra)?;
- let body = json_request::<SafekeeperUpsert>(&mut req).await?;
+ let mut req = match maybe_forward(req).await {
+ ForwardOutcome::Forwarded(res) => {
+ return res;
+ }
+ ForwardOutcome::NotForwarded(req) => req,
+ };
+
let id = parse_request_param::<i64>(&req, "id")?;
+ let service = get_state(&req).service.clone();
+ let strict = service.get_config().hakopod_ownership_v1;
+ let (body, ownership) = if strict {
+ let (body, ownership) = owned_json_request::<SafekeeperUpsert, _>(
+ &mut req,
+ "safekeeper_registration",
+ |body| body.id.to_string(),
+ )
+ .await?;
+ (body, Some(ownership))
+ } else {
+ (json_request::<SafekeeperUpsert>(&mut req).await?, None)
+ };
if id != body.id {
// it should be repeated
@@ -1729,17 +2069,35 @@ async fn handle_upsert_safekeeper(mut req: Request<Body>) -> Result<Response<Bod
)));
}
- let req = match maybe_forward(req).await {
- ForwardOutcome::Forwarded(res) => {
- return res;
+ if let Some(ownership) = ownership {
+ match service
+ .reserve_ownership(
+ ownership.token,
+ "safekeeper_registration",
+ &id.to_string(),
+ &ownership.request_hash,
+ )
+ .await?
+ {
+ OwnershipDecision::Replay(result) => return json_response(StatusCode::OK, result),
+ OwnershipDecision::Gone => {
+ return Err(ApiError::Conflict(
+ "safekeeper registration is permanently tombstoned".to_string(),
+ ));
+ }
+ OwnershipDecision::Begin | OwnershipDecision::Resume => {
+ service.upsert_safekeeper(body).await?;
+ let described = service.get_safekeeper(id).await?;
+ let result = ownership_result(described, ownership.token)?;
+ service
+ .complete_ownership(ownership.token, result.clone())
+ .await?;
+ return json_response(StatusCode::OK, result);
+ }
}
- ForwardOutcome::NotForwarded(req) => req,
- };
-
- let state = get_state(&req);
-
- state.service.upsert_safekeeper(body).await?;
+ }
+ service.upsert_safekeeper(body).await?;
Ok(Response::builder()
.status(StatusCode::NO_CONTENT)
.body(Body::empty())
@@ -1917,6 +2275,73 @@ fn epilogue_metrics_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>
})
}
+fn is_strict_ownership_route(method: &hyper::Method, path: &str) -> bool {
+ let segments = path.trim_matches('/').split('/').collect::<Vec<_>>();
+ match (method, segments.as_slice()) {
+ // This authenticated read operation uses POST because it accepts a
+ // structured inspection request. Hakopod needs it to verify the exact
+ // tenant attachment before an owned mutation or deletion.
+ (&hyper::Method::POST, ["debug", "v1", "inspect"]) => true,
+ (&hyper::Method::POST, ["v1", "tenant"]) => true,
+ (&hyper::Method::POST, ["v1", "tenant", _, "timeline"]) => true,
+ (&hyper::Method::DELETE, ["v1", "tenant", _]) => true,
+ (&hyper::Method::DELETE, ["v1", "tenant", _, "timeline", _]) => true,
+ (&hyper::Method::POST, ["control", "v1", "node"]) => true,
+ (&hyper::Method::POST, ["control", "v1", "safekeeper", _]) => true,
+ _ => false,
+ }
+}
+
+fn is_strict_upcall_route(method: &hyper::Method, path: &str) -> bool {
+ matches!(
+ (method, path),
+ (&hyper::Method::POST, "/upcall/v1/re-attach")
+ | (&hyper::Method::POST, "/upcall/v1/validate")
+ | (&hyper::Method::POST, "/upcall/v1/timeline_import_status")
+ )
+}
+
+fn ownership_route_middleware<B: hyper::body::HttpBody + Send + Sync + 'static>()
+-> Middleware<B, ApiError> {
+ Middleware::pre(move |req| async move {
+ let state = get_state(&req);
+ if !state.service.get_config().hakopod_ownership_v1
+ || req.method() == &hyper::Method::GET
+ || req.method() == &hyper::Method::HEAD
+ || is_strict_upcall_route(req.method(), req.uri().path())
+ || is_strict_ownership_route(req.method(), req.uri().path())
+ {
+ return Ok(req);
+ }
+ Err(ApiError::Forbidden(
+ "mutation route is disabled by Hakopod ownership strict mode".to_string(),
+ ))
+ })
+}
+
+async fn handle_ownership_capability(req: Request<Body>) -> Result<Response<Body>, ApiError> {
+ check_permissions(&req, Scope::Admin)?;
+ let state = get_state(&req);
+ if !state.service.get_config().hakopod_ownership_v1 {
+ return Err(ApiError::NotFound(
+ anyhow::anyhow!("Hakopod ownership protocol is disabled").into(),
+ ));
+ }
+ json_response(
+ StatusCode::OK,
+ serde_json::json!({
+ "protocol": "hakopod-ownership-v1",
+ "tenant_delete_protocol": "prepare-v1",
+ "mutations": [
+ "tenant",
+ "timeline",
+ "pageserver_registration",
+ "safekeeper_registration"
+ ]
+ }),
+ )
+}
+
pub async fn measured_metrics_handler(req: Request<Body>) -> Result<Response<Body>, ApiError> {
pub const TEXT_FORMAT: &str = "text/plain; version=0.0.4";
@@ -2188,6 +2613,7 @@ pub fn make_router(
build_info: BuildInfo,
) -> RouterBuilder<hyper::Body, ApiError> {
let mut router = endpoint::make_router()
+ .middleware(ownership_route_middleware())
.middleware(prologue_leadership_status_check_middleware())
.middleware(prologue_metrics_middleware())
.middleware(epilogue_metrics_middleware());
@@ -2217,6 +2643,13 @@ pub fn make_router(
.get("/ready", |r| {
named_request_span(r, handle_ready, RequestName("ready"))
})
+ .get("/control/v1/hakopod/ownership", |r| {
+ named_request_span(
+ r,
+ handle_ownership_capability,
+ RequestName("control_v1_hakopod_ownership"),
+ )
+ })
.get("/profile/cpu", |r| {
named_request_span(r, profile_cpu_handler, RequestName("profile_cpu"))
})
@@ -2684,7 +3117,84 @@ pub fn make_router(
#[cfg(test)]
mod test {
- use super::path_without_ids;
+ use super::{
+ is_strict_ownership_route, is_strict_upcall_route, owned_json_request, path_without_ids,
+ read_owned_request_body,
+ };
+ use bytes::Bytes;
+ use http_utils::error::ApiError;
+ use hyper::Body;
+ use std::time::Duration;
+
+ #[test]
+ fn strict_ownership_allows_authenticated_attachment_inspection() {
+ assert!(is_strict_ownership_route(
+ &hyper::Method::POST,
+ "/debug/v1/inspect"
+ ));
+ assert!(!is_strict_ownership_route(
+ &hyper::Method::POST,
+ "/debug/v1/node/1/drop"
+ ));
+ }
+
+ #[test]
+ fn strict_ownership_upcalls_are_exactly_allowlisted() {
+ assert!(is_strict_upcall_route(
+ &hyper::Method::POST,
+ "/upcall/v1/re-attach"
+ ));
+ assert!(is_strict_upcall_route(
+ &hyper::Method::POST,
+ "/upcall/v1/validate"
+ ));
+ assert!(!is_strict_upcall_route(
+ &hyper::Method::DELETE,
+ "/upcall/v1/re-attach"
+ ));
+ assert!(!is_strict_upcall_route(
+ &hyper::Method::POST,
+ "/upcall/v1/unknown"
+ ));
+ }
+
+ #[tokio::test]
+ async fn owned_request_body_rejects_streamed_oversize_payload() {
+ let chunks = futures::stream::iter([
+ Ok::<_, std::io::Error>(Bytes::from_static(b"1234")),
+ Ok(Bytes::from_static(b"5")),
+ ]);
+ let mut body = Body::wrap_stream(chunks);
+ assert!(matches!(
+ read_owned_request_body(&mut body, 4, Duration::from_secs(1)).await,
+ Err(ApiError::BadRequest(_))
+ ));
+ }
+
+ #[tokio::test]
+ async fn owned_request_body_times_out_stalled_stream() {
+ let mut body =
+ Body::wrap_stream(futures::stream::pending::<Result<Bytes, std::io::Error>>());
+ assert!(matches!(
+ read_owned_request_body(&mut body, 4, Duration::from_millis(10)).await,
+ Err(ApiError::Timeout(_))
+ ));
+ }
+
+ #[tokio::test]
+ async fn owned_request_rejects_missing_token_before_reading_body() {
+ let body = Body::wrap_stream(futures::stream::pending::<Result<Bytes, std::io::Error>>());
+ let mut request = hyper::Request::new(body);
+ let result = tokio::time::timeout(
+ Duration::from_millis(100),
+ owned_json_request::<serde_json::Value, _>(&mut request, "tenant", |_| {
+ "tenant-id".to_string()
+ }),
+ )
+ .await
+ .expect("missing token must be rejected before reading the body");
+ assert!(matches!(result, Err(ApiError::BadRequest(_))));
+ }
#[test]
fn test_path_without_ids() {
diff --git a/storage_controller/src/lib.rs b/storage_controller/src/lib.rs
index 24b06da83a25b5e939c5d7fbc9b3ac5300e8ed93..9efaa88a157bc5b54dc7e771d67454a80e0ca226 100644
--- a/storage_controller/src/lib.rs
+++ b/storage_controller/src/lib.rs
@@ -14,6 +14,7 @@
pub mod metrics;
mod node;
mod operation_utils;
+mod ownership;
mod pageserver_client;
mod peer_client;
pub mod persistence;
diff --git a/storage_controller/src/main.rs b/storage_controller/src/main.rs
index 34d4ac6fba5a4242e5a45e4df19a949445dfe5a2..f6b4d285794af700801e156fdaf85d7e0208cad4 100644
--- a/storage_controller/src/main.rs
+++ b/storage_controller/src/main.rs
@@ -102,6 +102,11 @@ struct Cli {
#[arg(long, default_value = "false")]
dev: bool,
+ /// Require provider-owned idempotency for all externally mutable database resources.
+ /// This is opt-in so existing Neon deployments and neon_local keep their legacy API.
+ #[arg(long, default_value = "false")]
+ hakopod_ownership_v1: bool,
+
/// Grace period before marking unresponsive pageserver offline
#[arg(long)]
max_offline_interval: Option<humantime::Duration>,
@@ -400,6 +405,9 @@ async fn async_main() -> anyhow::Result<()> {
"Running with less than 3 safekeepers per timeline is only permitted in `--dev` mode"
);
}
+ StrictMode::Dev if args.hakopod_ownership_v1 => {
+ anyhow::bail!("`--hakopod-ownership-v1` cannot be combined with `--dev`");
+ }
StrictMode::Strict => {
tracing::info!("Starting in strict mode: configuration is OK.")
}
@@ -430,6 +438,7 @@ async fn async_main() -> anyhow::Result<()> {
};
let config = Config {
+ hakopod_ownership_v1: args.hakopod_ownership_v1,
pageserver_jwt_token: secrets.pageserver_jwt_token,
safekeeper_jwt_token: secrets.safekeeper_jwt_token,
control_plane_jwt_token: secrets.control_plane_jwt_token,
diff --git a/storage_controller/src/ownership.rs b/storage_controller/src/ownership.rs
new file mode 100644
index 0000000000000000000000000000000000000000..6f3b3bf5bf1bed2e7f6c5e6c5da04942bb62ced8
--- /dev/null
+++ b/storage_controller/src/ownership.rs
@@ -0,0 +1,128 @@
+use http_utils::error::ApiError;
+use hyper::{Body, Request};
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+
+pub(crate) const HEADER: &str = "Hakopod-Ownership-Token";
+pub(crate) const DELETE_HEADER: &str = "Hakopod-Deletion-Token";
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct OwnershipRequest {
+ pub token: Uuid,
+ pub request_hash: [u8; 32],
+}
+
+pub(crate) fn parse(
+ request: &Request<Body>,
+ resource_kind: &str,
+ external_key: &str,
+ mut value: Value,
+) -> Result<OwnershipRequest, ApiError> {
+ let token = token(request)?;
+ canonicalize(&mut value);
+ let encoded =
+ serde_json::to_vec(&value).map_err(|e| ApiError::InternalServerError(e.into()))?;
+ let mut hasher = Sha256::new();
+ hasher.update(resource_kind.as_bytes());
+ hasher.update([0]);
+ hasher.update(external_key.as_bytes());
+ hasher.update([0]);
+ hasher.update(encoded);
+ Ok(OwnershipRequest {
+ token,
+ request_hash: hasher.finalize().into(),
+ })
+}
+
+pub(crate) fn token(request: &Request<Body>) -> Result<Uuid, ApiError> {
+ exact_token_header(request, HEADER)
+}
+
+pub(crate) fn delete_token(request: &Request<Body>) -> Result<Uuid, ApiError> {
+ exact_token_header(request, DELETE_HEADER)
+}
+
+fn exact_token_header(request: &Request<Body>, header: &str) -> Result<Uuid, ApiError> {
+ let raw_token = request
+ .headers()
+ .get(header)
+ .and_then(|v| v.to_str().ok())
+ .ok_or_else(|| ApiError::BadRequest(anyhow::anyhow!("missing or invalid Hakopod token")))?;
+ if raw_token.len() != 32
+ || raw_token == "00000000000000000000000000000000"
+ || !raw_token
+ .bytes()
+ .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
+ {
+ return Err(ApiError::BadRequest(anyhow::anyhow!(
+ "missing or invalid Hakopod token"
+ )));
+ }
+ Uuid::parse_str(raw_token)
+ .map_err(|_| ApiError::BadRequest(anyhow::anyhow!("missing or invalid Hakopod token")))
+}
+
+fn canonicalize(value: &mut Value) {
+ match value {
+ Value::Array(values) => values.iter_mut().for_each(canonicalize),
+ Value::Object(map) => {
+ let old = std::mem::take(map);
+ for (key, mut value) in old
+ .into_iter()
+ .collect::<std::collections::BTreeMap<_, _>>()
+ {
+ canonicalize(&mut value);
+ map.insert(key, value);
+ }
+ }
+ _ => {}
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ fn request(token: &str) -> Request<Body> {
+ let mut request = Request::new(Body::empty());
+ request.headers_mut().insert(HEADER, token.parse().unwrap());
+ request
+ }
+
+ #[test]
+ fn matches_go_contract_vector_and_ignores_object_key_order() {
+ let req = request("33333333333333333333333333333333");
+ let a = parse(&req, "tenant", "key", serde_json::json!({"a": 1, "b": 2})).unwrap();
+ let b = parse(&req, "tenant", "key", serde_json::json!({"b": 2, "a": 1})).unwrap();
+ assert_eq!(a.request_hash, b.request_hash);
+ assert_eq!(
+ hex::encode(a.request_hash),
+ "296002ae586b708f973d4a7f91c53789f83888e43daacbf1e08b645061ed87e5"
+ );
+ }
+
+ #[test]
+ fn hash_binds_kind_key_and_body() {
+ let req = request("33333333333333333333333333333333");
+ let a = parse(&req, "tenant", "key", serde_json::json!({"id": 1})).unwrap();
+ let b = parse(&req, "tenant", "other", serde_json::json!({"id": 1})).unwrap();
+ let c = parse(&req, "timeline", "key", serde_json::json!({"id": 1})).unwrap();
+ let d = parse(&req, "tenant", "key", serde_json::json!({"id": 2})).unwrap();
+ assert_ne!(a.request_hash, b.request_hash);
+ assert_ne!(a.request_hash, c.request_hash);
+ assert_ne!(a.request_hash, d.request_hash);
+ }
+
+ #[test]
+ fn token_encoding_is_exact() {
+ for invalid in [
+ "00000000000000000000000000000000",
+ "33333333-3333-3333-3333-333333333333",
+ "3333333333333333333333333333333G",
+ "3333333333333333333333333333333",
+ ] {
+ let request = request(invalid);
+ assert!(parse(&request, "tenant", "key", Value::Null).is_err());
+ }
+ }
+}
diff --git a/storage_controller/src/pageserver_client.rs b/storage_controller/src/pageserver_client.rs
index 9e829e252de6dc4ee05debe8b01ec1a78c3e1d01..775e2e083390d24eb2a3e057a44c6d39c1ff510c 100644
--- a/storage_controller/src/pageserver_client.rs
+++ b/storage_controller/src/pageserver_client.rs
@@ -63,12 +63,31 @@ pub(crate) fn new(
}
}
- pub(crate) async fn tenant_delete(&self, tenant_shard_id: TenantShardId) -> Result<StatusCode> {
+ pub(crate) async fn tenant_delete(
+ &self,
+ tenant_shard_id: TenantShardId,
+ request: &pageserver_api::models::TenantDeleteRequest,
+ ) -> Result<StatusCode> {
measured_request!(
"tenant",
crate::metrics::Method::Delete,
&self.node_id_label,
- self.inner.tenant_delete(tenant_shard_id).await
+ self.inner.tenant_delete(tenant_shard_id, request).await
+ )
+ }
+
+ pub(crate) async fn tenant_delete_preflight(
+ &self,
+ tenant_shard_id: TenantShardId,
+ request: &pageserver_api::models::TenantDeleteRequest,
+ ) -> Result<pageserver_api::models::TenantDeletePreflightResponse> {
+ measured_request!(
+ "tenant_delete_preflight",
+ crate::metrics::Method::Delete,
+ &self.node_id_label,
+ self.inner
+ .tenant_delete_preflight(tenant_shard_id, request)
+ .await
)
}
@@ -242,13 +261,14 @@ pub(crate) async fn timeline_delete(
&self,
tenant_shard_id: TenantShardId,
timeline_id: TimelineId,
+ ownership_token: Option<&str>,
) -> Result<StatusCode> {
measured_request!(
"timeline",
crate::metrics::Method::Delete,
&self.node_id_label,
self.inner
- .timeline_delete(tenant_shard_id, timeline_id)
+ .timeline_delete(tenant_shard_id, timeline_id, ownership_token)
.await
)
}
diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs
index c61ef9ff5d2db45049839ca8a89c7c154a5316b6..0693dcd824882ab63de96b81b64b8325350e97c7 100644
--- a/storage_controller/src/persistence.rs
+++ b/storage_controller/src/persistence.rs
@@ -32,9 +32,11 @@
use safekeeper_api::membership::SafekeeperGeneration;
use scoped_futures::ScopedBoxFuture;
use serde::{Deserialize, Serialize};
+use serde_json::Value as JsonValue;
use utils::generation::Generation;
use utils::id::{NodeId, TenantId, TimelineId};
use utils::lsn::Lsn;
+use uuid::Uuid;
use self::split_state::SplitState;
use crate::metrics::{
@@ -77,6 +79,406 @@ pub struct Persistence {
connection_pool: Pool<AsyncPgConnection>,
}
+impl Persistence {
+ pub(crate) async fn reserve_ownership(
+ &self,
+ token: Uuid,
+ resource_kind: &str,
+ external_key: &str,
+ request_hash: &[u8; 32],
+ ) -> DatabaseResult<OwnershipDecision> {
+ let resource_kind = resource_kind.to_owned();
+ let external_key = external_key.to_owned();
+ let request_hash = request_hash.to_vec();
+ self.with_conn(move |conn| {
+ let resource_kind = resource_kind.clone();
+ let external_key = external_key.clone();
+ let request_hash = request_hash.clone();
+ Box::pin(async move {
+ if resource_kind == "timeline" {
+ let (tenant_id, _) = external_key.split_once('/').ok_or_else(|| {
+ DatabaseError::Cas("invalid timeline ownership key".into())
+ })?;
+ let parent = diesel::sql_query(
+ "SELECT state::text AS state FROM hakopod_resource_ownership \
+ WHERE resource_kind = 'tenant' AND external_key = $1 FOR SHARE",
+ )
+ .bind::<diesel::sql_types::Text, _>(tenant_id)
+ .get_result::<OwnershipStateRow>(conn)
+ .await
+ .optional()?;
+ match parent.as_ref().map(|row| row.state.as_str()) {
+ Some("completed") => {}
+ Some("deleting" | "tombstoned") => {
+ return Err(DatabaseError::Cas(
+ "tenant ownership is permanently deleting".into(),
+ ));
+ }
+ Some(_) => {
+ return Err(DatabaseError::Cas(
+ "tenant ownership is not ready for timeline creation".into(),
+ ));
+ }
+ None => {
+ return Err(DatabaseError::Cas(
+ "tenant has no durable ownership record".into(),
+ ));
+ }
+ }
+ }
+ let inserted = diesel::sql_query(
+ "INSERT INTO hakopod_resource_ownership \
+ (ownership_token, resource_kind, external_key, request_hash, state) \
+ VALUES ($1, $2, $3, $4, 'reserved') ON CONFLICT DO NOTHING",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .bind::<diesel::sql_types::Text, _>(&resource_kind)
+ .bind::<diesel::sql_types::Text, _>(&external_key)
+ .bind::<diesel::sql_types::Binary, _>(&request_hash)
+ .execute(conn)
+ .await?;
+
+ // A newly inserted ledger row must never adopt provider state
+ // that predates the ownership record. This check shares the
+ // SERIALIZABLE transaction with the reservation, so a racing
+ // provider-table insert forces one transaction to retry or
+ // fail instead of silently converting a legacy resource.
+ if inserted == 1 {
+ let exists = match resource_kind.as_str() {
+ "tenant" => diesel::sql_query(
+ "SELECT EXISTS(SELECT 1 FROM tenant_shards WHERE tenant_id = $1) AS exists",
+ )
+ .bind::<diesel::sql_types::Text, _>(&external_key)
+ .get_result::<ExistsRow>(conn)
+ .await?
+ .exists,
+ "timeline" => {
+ let (tenant_id, timeline_id) = external_key.split_once('/').ok_or_else(|| {
+ DatabaseError::Cas("invalid timeline ownership key".into())
+ })?;
+ diesel::sql_query(
+ "SELECT EXISTS(SELECT 1 FROM timelines WHERE tenant_id = $1 AND timeline_id = $2) AS exists",
+ )
+ .bind::<diesel::sql_types::Text, _>(tenant_id)
+ .bind::<diesel::sql_types::Text, _>(timeline_id)
+ .get_result::<ExistsRow>(conn)
+ .await?
+ .exists
+ }
+ "pageserver_registration" => {
+ let node_id = external_key.parse::<i64>().map_err(|_| {
+ DatabaseError::Cas("invalid pageserver ownership key".into())
+ })?;
+ diesel::sql_query(
+ "SELECT EXISTS(SELECT 1 FROM nodes WHERE node_id = $1) AS exists",
+ )
+ .bind::<diesel::sql_types::BigInt, _>(node_id)
+ .get_result::<ExistsRow>(conn)
+ .await?
+ .exists
+ }
+ "safekeeper_registration" => {
+ let node_id = external_key.parse::<i64>().map_err(|_| {
+ DatabaseError::Cas("invalid safekeeper ownership key".into())
+ })?;
+ diesel::sql_query(
+ "SELECT EXISTS(SELECT 1 FROM safekeepers WHERE id = $1) AS exists",
+ )
+ .bind::<diesel::sql_types::BigInt, _>(node_id)
+ .get_result::<ExistsRow>(conn)
+ .await?
+ .exists
+ }
+ _ => {
+ return Err(DatabaseError::Cas(
+ "unsupported ownership resource kind".into(),
+ ));
+ }
+ };
+ if exists {
+ return Err(DatabaseError::Cas(
+ "refusing to adopt a pre-existing provider resource".into(),
+ ));
+ }
+ }
+
+ let mut rows = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1 OR (resource_kind = $2 AND external_key = $3) \
+ FOR UPDATE",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .bind::<diesel::sql_types::Text, _>(&resource_kind)
+ .bind::<diesel::sql_types::Text, _>(&external_key)
+ .load::<OwnershipRow>(conn)
+ .await?;
+ if rows.len() != 1 {
+ return Err(DatabaseError::Cas("ownership token or external key conflicts with another resource".into()));
+ }
+ let row = rows.pop().expect("one ownership row");
+ if row.ownership_token != token
+ || row.resource_kind != resource_kind
+ || row.external_key != external_key
+ || row.request_hash != request_hash
+ {
+ return Err(DatabaseError::Cas("ownership token, key, or request hash conflicts".into()));
+ }
+ match row.state.as_str() {
+ "reserved" => {
+ let updated = diesel::sql_query(
+ "UPDATE hakopod_resource_ownership SET state = 'applying', \
+ updated_at = now() WHERE ownership_token = $1 AND state = 'reserved'",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .execute(conn)
+ .await?;
+ if updated == 1 {
+ Ok(OwnershipDecision::Begin)
+ } else {
+ let current = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .get_result::<OwnershipRow>(conn)
+ .await?;
+ ownership_resume_decision(current)
+ }
+ }
+ _ => ownership_resume_decision(row),
+ }
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn complete_ownership(
+ &self,
+ token: Uuid,
+ result: JsonValue,
+ ) -> DatabaseResult<()> {
+ self.with_conn(move |conn| {
+ let result = result.clone();
+ Box::pin(async move {
+ let rows = diesel::sql_query(
+ "UPDATE hakopod_resource_ownership SET state = 'completed', result = $2, \
+ error_code = NULL, updated_at = now() WHERE ownership_token = $1 \
+ AND state IN ('reserved', 'applying')",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .bind::<diesel::sql_types::Jsonb, _>(&result)
+ .execute(conn)
+ .await?;
+ if rows == 1 {
+ return Ok(());
+ }
+ let row = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1 FOR UPDATE",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .get_result::<OwnershipRow>(conn)
+ .await?;
+ if row.state == "completed" && row.result.as_ref() == Some(&result) {
+ Ok(())
+ } else {
+ Err(DatabaseError::Cas(
+ "ownership completion conflicts with durable state".into(),
+ ))
+ }
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn begin_ownership_delete(
+ &self,
+ token: Uuid,
+ resource_kind: &str,
+ external_key: &str,
+ ) -> DatabaseResult<OwnershipDecision> {
+ let resource_kind = resource_kind.to_owned();
+ let external_key = external_key.to_owned();
+ self.with_conn(move |conn| {
+ let resource_kind = resource_kind.clone();
+ let external_key = external_key.clone();
+ Box::pin(async move {
+ let row = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE resource_kind = $1 AND external_key = $2 FOR UPDATE",
+ )
+ .bind::<diesel::sql_types::Text, _>(&resource_kind)
+ .bind::<diesel::sql_types::Text, _>(&external_key)
+ .get_result::<OwnershipRow>(conn)
+ .await
+ .optional()?
+ .ok_or_else(|| {
+ DatabaseError::Cas("resource has no durable ownership record".into())
+ })?;
+ if row.ownership_token != token {
+ return Err(DatabaseError::Cas("delete ownership token does not match".into()));
+ }
+ match row.state.as_str() {
+ "completed" => {
+ let updated = diesel::sql_query(
+ "UPDATE hakopod_resource_ownership SET state = 'deleting', \
+ updated_at = now() WHERE ownership_token = $1 AND state = 'completed'",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .execute(conn)
+ .await?;
+ if updated == 1 {
+ Ok(OwnershipDecision::Begin)
+ } else {
+ let current = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .get_result::<OwnershipRow>(conn)
+ .await?;
+ ownership_delete_decision(current)
+ }
+ }
+ _ => ownership_delete_decision(row),
+ }
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn finish_ownership_delete(&self, token: Uuid) -> DatabaseResult<()> {
+ self.with_conn(move |conn| {
+ Box::pin(async move {
+ let rows = diesel::sql_query(
+ "UPDATE hakopod_resource_ownership SET state = 'tombstoned', \
+ updated_at = now() WHERE ownership_token = $1 AND state = 'deleting'",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .execute(conn)
+ .await?;
+ if rows == 1 {
+ return Ok(());
+ }
+ let row = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .get_result::<OwnershipRow>(conn)
+ .await?;
+ if row.state == "tombstoned" {
+ Ok(())
+ } else {
+ Err(DatabaseError::Cas(
+ "ownership tombstone conflicts with durable state".into(),
+ ))
+ }
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn set_ownership_delete_token(
+ &self,
+ token: Uuid,
+ delete_token: Uuid,
+ ) -> DatabaseResult<()> {
+ self.with_conn(move |conn| {
+ Box::pin(async move {
+ let rows = diesel::sql_query(
+ "UPDATE hakopod_resource_ownership SET delete_token = $2, updated_at = now() \
+ WHERE ownership_token = $1 AND state = 'deleting' \
+ AND (delete_token IS NULL OR delete_token = $2)",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .bind::<diesel::sql_types::Uuid, _>(delete_token)
+ .execute(conn)
+ .await?;
+ if rows == 1 {
+ return Ok(());
+ }
+ let row = diesel::sql_query(
+ "SELECT delete_token FROM hakopod_resource_ownership \
+ WHERE ownership_token = $1 AND state IN ('deleting', 'tombstoned')",
+ )
+ .bind::<diesel::sql_types::Uuid, _>(token)
+ .get_result::<OwnershipDeleteTokenRow>(conn)
+ .await
+ .optional()?;
+ if row.and_then(|row| row.delete_token) == Some(delete_token) {
+ Ok(())
+ } else {
+ Err(DatabaseError::Cas(
+ "ownership delete token conflicts with durable state".into(),
+ ))
+ }
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn get_ownership(
+ &self,
+ resource_kind: &str,
+ external_key: &str,
+ ) -> DatabaseResult<Option<OwnershipRecord>> {
+ let resource_kind = resource_kind.to_owned();
+ let external_key = external_key.to_owned();
+ self.with_conn(move |conn| {
+ let resource_kind = resource_kind.clone();
+ let external_key = external_key.clone();
+ Box::pin(async move {
+ let row = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE resource_kind = $1 AND external_key = $2",
+ )
+ .bind::<diesel::sql_types::Text, _>(&resource_kind)
+ .bind::<diesel::sql_types::Text, _>(&external_key)
+ .get_result::<OwnershipRow>(conn)
+ .await
+ .optional()?;
+ Ok(row.map(OwnershipRecord::from))
+ })
+ })
+ .await
+ }
+
+ pub(crate) async fn list_timeline_ownership_for_tenant(
+ &self,
+ tenant_id: TenantId,
+ limit: i64,
+ ) -> DatabaseResult<Vec<OwnershipRecord>> {
+ let tenant_id = tenant_id.to_string();
+ self.with_conn(move |conn| {
+ let tenant_id = tenant_id.clone();
+ Box::pin(async move {
+ let rows = diesel::sql_query(
+ "SELECT ownership_token, resource_kind, external_key, request_hash, \
+ state::text AS state, result, delete_token FROM hakopod_resource_ownership \
+ WHERE resource_kind = 'timeline' \
+ AND external_key >= $1 || '/' AND external_key < $1 || '0' \
+ AND state <> 'tombstoned' \
+ ORDER BY external_key LIMIT $2",
+ )
+ .bind::<diesel::sql_types::Text, _>(&tenant_id)
+ .bind::<diesel::sql_types::BigInt, _>(limit)
+ .load::<OwnershipRow>(conn)
+ .await?;
+ Ok(rows.into_iter().map(OwnershipRecord::from).collect())
+ })
+ })
+ .await
+ }
+}
+
/// Legacy format, for use in JSON compat objects in test environment
#[derive(Serialize, Deserialize)]
struct JsonPersistence {
@@ -155,6 +557,101 @@ pub(crate) enum AbortShardSplitStatus {
pub(crate) type DatabaseResult<T> = Result<T, DatabaseError>;
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum OwnershipDecision {
+ Begin,
+ Resume,
+ Replay(JsonValue),
+ Gone,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct OwnershipRecord {
+ pub(crate) token: Uuid,
+ pub(crate) resource_kind: String,
+ pub(crate) external_key: String,
+ pub(crate) request_hash: Vec<u8>,
+ pub(crate) state: String,
+ pub(crate) result: Option<JsonValue>,
+ pub(crate) delete_token: Option<Uuid>,
+}
+
+#[derive(QueryableByName)]
+struct OwnershipRow {
+ #[diesel(sql_type = diesel::sql_types::Uuid)]
+ ownership_token: Uuid,
+ #[diesel(sql_type = diesel::sql_types::Text)]
+ resource_kind: String,
+ #[diesel(sql_type = diesel::sql_types::Text)]
+ external_key: String,
+ #[diesel(sql_type = diesel::sql_types::Binary)]
+ request_hash: Vec<u8>,
+ #[diesel(sql_type = diesel::sql_types::Text)]
+ state: String,
+ #[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Jsonb>)]
+ result: Option<JsonValue>,
+ #[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Uuid>)]
+ delete_token: Option<Uuid>,
+}
+
+#[derive(QueryableByName)]
+struct OwnershipDeleteTokenRow {
+ #[diesel(sql_type = diesel::sql_types::Nullable<diesel::sql_types::Uuid>)]
+ delete_token: Option<Uuid>,
+}
+
+#[derive(QueryableByName)]
+struct ExistsRow {
+ #[diesel(sql_type = diesel::sql_types::Bool)]
+ exists: bool,
+}
+
+#[derive(QueryableByName)]
+struct OwnershipStateRow {
+ #[diesel(sql_type = diesel::sql_types::Text)]
+ state: String,
+}
+
+fn ownership_resume_decision(row: OwnershipRow) -> DatabaseResult<OwnershipDecision> {
+ match row.state.as_str() {
+ "reserved" | "applying" => Ok(OwnershipDecision::Resume),
+ "completed" => row.result.map(OwnershipDecision::Replay).ok_or_else(|| {
+ DatabaseError::Logical("completed ownership record has no result".into())
+ }),
+ "deleting" | "tombstoned" => Ok(OwnershipDecision::Gone),
+ "failed" => Err(DatabaseError::Cas(
+ "ownership mutation is permanently failed".into(),
+ )),
+ state => Err(DatabaseError::Logical(format!(
+ "unknown ownership state {state:?}"
+ ))),
+ }
+}
+
+fn ownership_delete_decision(row: OwnershipRow) -> DatabaseResult<OwnershipDecision> {
+ match row.state.as_str() {
+ "deleting" => Ok(OwnershipDecision::Resume),
+ "tombstoned" => Ok(OwnershipDecision::Gone),
+ _ => Err(DatabaseError::Cas(
+ "resource is not in a deletable ownership state".into(),
+ )),
+ }
+}
+
+impl From<OwnershipRow> for OwnershipRecord {
+ fn from(row: OwnershipRow) -> Self {
+ Self {
+ token: row.ownership_token,
+ resource_kind: row.resource_kind,
+ external_key: row.external_key,
+ request_hash: row.request_hash,
+ state: row.state,
+ result: row.result,
+ delete_token: row.delete_token,
+ }
+ }
+}
+
/// Some methods can operate on either a whole tenant or a single shard
#[derive(Clone)]
pub(crate) enum TenantFilter {
@@ -1785,6 +2282,34 @@ pub(crate) async fn list_timelines_for_tenant(
Ok(timelines)
}
+ pub(crate) async fn list_timelines_for_tenant_bounded(
+ &self,
+ tenant_id: TenantId,
+ limit: i64,
+ ) -> DatabaseResult<Vec<TimelinePersistence>> {
+ use crate::schema::timelines::dsl;
+
+ let tenant_id = &tenant_id;
+ let timelines = self
+ .with_measured_conn(DatabaseOperation::GetTimeline, move |conn| {
+ Box::pin(async move {
+ let timelines: Vec<TimelineFromDb> = dsl::timelines
+ .filter(dsl::tenant_id.eq(&tenant_id.to_string()))
+ .order(dsl::timeline_id.asc())
+ .limit(limit)
+ .load(conn)
+ .await?;
+ Ok(timelines)
+ })
+ })
+ .await?;
+
+ Ok(timelines
+ .into_iter()
+ .map(TimelineFromDb::into_persistence)
+ .collect())
+ }
+
/// Persist pending op. Returns if it was newly inserted. If it wasn't, we haven't done any writes.
pub(crate) async fn insert_pending_op(
&self,
@@ -2611,6 +3136,8 @@ pub(crate) struct TimelinePersistence {
pub(crate) cplane_notified_generation: i32,
pub(crate) deleted_at: Option<chrono::DateTime<chrono::Utc>>,
pub(crate) sk_set_notified_generation: i32,
+ pub(crate) ownership_token: Option<String>,
+ pub(crate) ownership_request_hash: Option<String>,
}
/// This is separate from [TimelinePersistence] only because postgres allows NULLs
@@ -2630,6 +3157,8 @@ pub(crate) struct TimelineFromDb {
pub(crate) cplane_notified_generation: i32,
pub(crate) deleted_at: Option<chrono::DateTime<chrono::Utc>>,
pub(crate) sk_set_notified_generation: i32,
+ pub(crate) ownership_token: Option<String>,
+ pub(crate) ownership_request_hash: Option<String>,
}
impl TimelineFromDb {
@@ -2650,6 +3179,8 @@ fn into_persistence(self) -> TimelinePersistence {
cplane_notified_generation: self.cplane_notified_generation,
deleted_at: self.deleted_at,
sk_set_notified_generation: self.sk_set_notified_generation,
+ ownership_token: self.ownership_token,
+ ownership_request_hash: self.ownership_request_hash,
}
}
}
diff --git a/storage_controller/src/safekeeper_client.rs b/storage_controller/src/safekeeper_client.rs
index 47a785e7d34a1946aa8b8931e86b299a3e7a3d1a..4501850bfd3dad97843eb24cbc79100f80b873b2 100644
--- a/storage_controller/src/safekeeper_client.rs
+++ b/storage_controller/src/safekeeper_client.rs
@@ -1,5 +1,6 @@
use safekeeper_api::models::{
- self, PullTimelineRequest, PullTimelineResponse, SafekeeperUtilization, TimelineCreateRequest,
+ self, OwnershipValidationRequest, OwnershipValidationResponse, PullTimelineRequest,
+ PullTimelineResponse, SafekeeperUtilization, TimelineCreateRequest,
};
use safekeeper_client::mgmt_api::{Client, Result};
use utils::id::{NodeId, TenantId, TimelineId};
@@ -72,6 +73,18 @@ pub(crate) async fn create_timeline(
)
}
+ pub(crate) async fn validate_ownership(
+ &self,
+ req: &OwnershipValidationRequest,
+ ) -> Result<OwnershipValidationResponse> {
+ measured_request!(
+ "validate_ownership",
+ crate::metrics::Method::Post,
+ &self.node_id_label,
+ self.inner.validate_ownership(req).await
+ )
+ }
+
#[allow(unused)]
pub(crate) async fn exclude_timeline(
&self,
@@ -93,12 +106,15 @@ pub(crate) async fn delete_timeline(
&self,
tenant_id: TenantId,
timeline_id: TimelineId,
+ ownership_token: Option<&str>,
) -> Result<models::TimelineDeleteResult> {
measured_request!(
"delete_timeline",
crate::metrics::Method::Delete,
&self.node_id_label,
- self.inner.delete_timeline(tenant_id, timeline_id).await
+ self.inner
+ .delete_timeline(tenant_id, timeline_id, ownership_token)
+ .await
)
}
diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs
index def519c168108e6131effa6d4330327c1fa6d67e..ab55ad84e94b5fc2fa00e2aa1c80deac1e9856bb 100644
--- a/storage_controller/src/schema.rs
+++ b/storage_controller/src/schema.rs
@@ -119,6 +119,8 @@ pub mod sql_types {
cplane_notified_generation -> Int4,
deleted_at -> Nullable<Timestamptz>,
sk_set_notified_generation -> Int4,
+ ownership_token -> Nullable<Text>,
+ ownership_request_hash -> Nullable<Text>,
}
}
diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs
index 2c7e4ee2def6390e1f7ce2ad553b2469d36c4b7b..925906ae100e78e573e9cb91d0b6a4eaf124cf45 100644
--- a/storage_controller/src/service.rs
+++ b/storage_controller/src/service.rs
@@ -36,8 +36,9 @@
};
use pageserver_api::models::{
self, DetachBehavior, LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease,
- PageserverUtilization, SecondaryProgress, ShardImportStatus, ShardParameters, TenantConfig,
- TenantConfigPatchRequest, TenantConfigRequest, TenantLocationConfigRequest,
+ OwnedTimelineDelete, PageserverUtilization, SecondaryProgress, ShardImportStatus,
+ ShardParameters, TenantConfig, TenantConfigPatchRequest, TenantConfigRequest,
+ TenantDeletePreflightResponse, TenantDeleteRequest, TenantLocationConfigRequest,
TenantLocationConfigResponse, TenantShardLocation, TenantShardSplitRequest,
TenantShardSplitResponse, TenantSorting, TenantTimeTravelRequest,
TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateResponseStorcon,
@@ -368,6 +369,9 @@ fn become_leader(&mut self) {
#[derive(Clone)]
pub struct Config {
+ /// Fence all externally mutable database resources with the Hakopod v1
+ /// provider ownership protocol. Legacy deployments leave this disabled.
+ pub hakopod_ownership_v1: bool,
// All pageservers managed by one instance of this service must have
// the same public key. This JWT token will be used to authenticate
// this service to the pageservers it manages.
@@ -773,6 +777,126 @@ pub fn get_config(&self) -> &Config {
&self.config
}
+ pub(crate) async fn reserve_ownership(
+ &self,
+ token: uuid::Uuid,
+ resource_kind: &str,
+ external_key: &str,
+ request_hash: &[u8; 32],
+ ) -> Result<crate::persistence::OwnershipDecision, ApiError> {
+ self.persistence
+ .reserve_ownership(token, resource_kind, external_key, request_hash)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn complete_ownership(
+ &self,
+ token: uuid::Uuid,
+ result: serde_json::Value,
+ ) -> Result<(), ApiError> {
+ self.persistence
+ .complete_ownership(token, result)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn begin_ownership_delete(
+ &self,
+ token: uuid::Uuid,
+ resource_kind: &str,
+ external_key: &str,
+ ) -> Result<crate::persistence::OwnershipDecision, ApiError> {
+ self.persistence
+ .begin_ownership_delete(token, resource_kind, external_key)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn finish_ownership_delete(&self, token: uuid::Uuid) -> Result<(), ApiError> {
+ self.persistence
+ .finish_ownership_delete(token)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn set_ownership_delete_token(
+ &self,
+ token: uuid::Uuid,
+ delete_token: uuid::Uuid,
+ ) -> Result<(), ApiError> {
+ self.persistence
+ .set_ownership_delete_token(token, delete_token)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn get_ownership(
+ &self,
+ resource_kind: &str,
+ external_key: &str,
+ ) -> Result<Option<crate::persistence::OwnershipRecord>, ApiError> {
+ self.persistence
+ .get_ownership(resource_kind, external_key)
+ .await
+ .map_err(Into::into)
+ }
+
+ pub(crate) async fn preflight_owned_timelines_for_tenant(
+ &self,
+ tenant_id: TenantId,
+ ) -> Result<Vec<(uuid::Uuid, OwnedTimelineDelete)>, ApiError> {
+ const MAX_OWNED_TIMELINES: usize = 4096;
+ let records = self
+ .persistence
+ .list_timeline_ownership_for_tenant(tenant_id, (MAX_OWNED_TIMELINES + 1) as i64)
+ .await?;
+ if records.len() > MAX_OWNED_TIMELINES {
+ return Err(ApiError::Conflict(format!(
+ "tenant has more than {MAX_OWNED_TIMELINES} owned timelines"
+ )));
+ }
+
+ records
+ .into_iter()
+ .map(|record| {
+ let (record_tenant_id, timeline_id) =
+ record.external_key.split_once('/').ok_or_else(|| {
+ ApiError::InternalServerError(anyhow::anyhow!(
+ "invalid owned timeline external key"
+ ))
+ })?;
+ if record_tenant_id != tenant_id.to_string() {
+ return Err(ApiError::Conflict(
+ "owned timeline belongs to another tenant".to_string(),
+ ));
+ }
+ let timeline_id = TimelineId::from_str(timeline_id)
+ .context("timeline id loaded from ownership ledger")
+ .map_err(ApiError::InternalServerError)?;
+ if !matches!(record.state.as_str(), "completed" | "deleting") {
+ return Err(ApiError::Conflict(format!(
+ "owned timeline {timeline_id} is in non-deletable state {}",
+ record.state
+ )));
+ }
+ let request_hash: [u8; 32] = record.request_hash.try_into().map_err(|_| {
+ ApiError::InternalServerError(anyhow::anyhow!(
+ "owned timeline {timeline_id} has an invalid request hash"
+ ))
+ })?;
+ Ok((
+