diff --git a/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs b/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs index 29d1c3949..2427783d0 100644 --- a/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs +++ b/crates/socket-patch-core/src/patch/redirect/upstream/pypi.rs @@ -781,27 +781,6 @@ fn restore_hatch_envs( } } -/// Remove `keys`' last table's `allow-direct-references = true` (the Hatch -/// permission the hosted rewrite set), then every table on the path that is -/// left empty. -fn drop_direct_reference_permission(doc: &mut DocumentMut, keys: &[&str]) -> bool { - fn walk(table: &mut dyn toml_edit::TableLike, keys: &[&str]) -> bool { - let Some((first, rest)) = keys.split_first() else { - return table.get("allow-direct-references").and_then(Item::as_bool) == Some(true) - && table.remove("allow-direct-references").is_some(); - }; - let Some(child) = table.get_mut(first).and_then(Item::as_table_like_mut) else { - return false; - }; - let removed = walk(child, rest); - if removed && child.is_empty() { - table.remove(first); - } - removed - } - walk(doc.as_table_mut(), keys) -} - pub(crate) async fn restore_hatch( view: &mut View<'_>, pins: &[&HostedPin], @@ -885,13 +864,16 @@ pub(crate) async fn restore_hatch( let external = docs .get("hatch.toml") .is_some_and(|(_, doc)| doc.contains_key("metadata")); - let (file, keys): (&str, &[&str]) = if external { - ("hatch.toml", &["metadata"]) + let file = if external { + "hatch.toml" } else { - ("pyproject.toml", &["tool", "hatch", "metadata"]) + "pyproject.toml" }; if let Some((_, doc)) = docs.get_mut(file) { - drop_direct_reference_permission(doc, keys); + crate::utils::hatch::drop_direct_reference_permission( + doc, + crate::utils::hatch::permission_keys(external), + ); } } let restored: BTreeSet = restored diff --git a/crates/socket-patch-core/src/utils/hatch.rs b/crates/socket-patch-core/src/utils/hatch.rs index 4be79627a..811e26796 100644 --- a/crates/socket-patch-core/src/utils/hatch.rs +++ b/crates/socket-patch-core/src/utils/hatch.rs @@ -265,8 +265,7 @@ fn rewrite_environments( .is_some_and(|kind| kind != "virtual") { return Err( - "Hatch sources, overrides and custom environments require agent mode" - .into(), + "Hatch sources, overrides and custom environments require agent mode".into(), ); } for key in ["dependencies", "extra-dependencies"] { @@ -303,11 +302,7 @@ pub fn rewrite( } fn enable_permission(document: &mut DocumentMut, external: bool) -> Result<(), String> { - let keys: &[&str] = if external { - &["metadata"] - } else { - &["tool", "hatch", "metadata"] - }; + let keys = permission_keys(external); let mut table: &mut dyn toml_edit::TableLike = document.as_table_mut(); for key in keys { if !table.contains_key(key) { @@ -328,6 +323,40 @@ fn enable_permission(document: &mut DocumentMut, external: bool) -> Result<(), S Ok(()) } +/// The table path holding Hatch's direct-reference permission: hatch.toml's +/// `[metadata]` when that file carries the key, else pyproject's +/// `[tool.hatch.metadata]`. +pub(crate) fn permission_keys(external: bool) -> &'static [&'static str] { + if external { + &["metadata"] + } else { + &["tool", "hatch", "metadata"] + } +} + +/// Remove `keys`' last table's `allow-direct-references = true` (the +/// permission a project direct reference needs), then every table on the +/// path that is left empty. Shared by the hosted unwind and the vendored +/// permission ledger, so both lanes agree on what the file looks like once +/// no project direct reference is left. +pub(crate) fn drop_direct_reference_permission(doc: &mut DocumentMut, keys: &[&str]) -> bool { + fn walk(table: &mut dyn toml_edit::TableLike, keys: &[&str]) -> bool { + let Some((first, rest)) = keys.split_first() else { + return table.get("allow-direct-references").and_then(Item::as_bool) == Some(true) + && table.remove("allow-direct-references").is_some(); + }; + let Some(child) = table.get_mut(first).and_then(Item::as_table_like_mut) else { + return false; + }; + let removed = walk(child, rest); + if removed && child.is_empty() { + table.remove(first); + } + removed + } + walk(doc.as_table_mut(), keys) +} + pub fn has_project_direct_references(files: &BTreeMap) -> bool { let Some(document) = parsed(files, HATCH_FILES[0]) else { return false; diff --git a/crates/socket-patch-core/src/vendor/pypi_hatch.rs b/crates/socket-patch-core/src/vendor/pypi_hatch.rs index a14464697..e1e94d4f8 100644 --- a/crates/socket-patch-core/src/vendor/pypi_hatch.rs +++ b/crates/socket-patch-core/src/vendor/pypi_hatch.rs @@ -195,12 +195,19 @@ pub(super) async fn wire( .find(|record| record.kind == "hatch_permission" && record.file == permission.file) .cloned() .unwrap_or_else(|| { + // A permission live references hold is recorded as the + // value it has once they are gone, the same rule the + // hosted unwind applies. + let original = permission_held_by_live_references(&project.files) + .then(|| drop_owned_permission(&permission.original, &permission.file)) + .flatten() + .unwrap_or(permission.original); record( &permission.file, "hatch_permission", WiringAction::Rewritten, "allow-direct-references", - Some(permission.original), + Some(original), permission.new, ) }), @@ -313,6 +320,41 @@ pub(super) async fn revert(entry: &VendorEntry, root: &Path, dry_run: bool) -> R RevertOutcome::ok() } +/// Whether `files` hold a project direct reference the vendored ledger does +/// not own, so any direct-reference permission there is held for it. A +/// hosted→vendored takeover unwinds hosted pins one at a time (#674), so +/// when the first package is vendored the other packages' hosted references +/// still hold the permission hosted mode added. Recording that state as the +/// permission's "original" would make rollback restore a permission the user +/// never had. Vendored references don't count: a permission they need was +/// recorded before them. +fn permission_held_by_live_references(files: &BTreeMap) -> bool { + files + .get(hatch::HATCH_FILES[0]) + .and_then(|text| text.trim_start_matches('\u{feff}').parse().ok()) + .is_some_and(|document: toml_edit::DocumentMut| { + crate::vendor::common::pyproject_dependency_specs(&document) + .into_iter() + .filter_map(|(_, spec)| spec.split(';').next()?.split_once('@')) + .any(|(_, location)| { + !location + .trim_start() + .starts_with("{root:uri}/.socket/vendor/") + }) + }) +} + +/// `text` without its direct-reference permission and the tables that +/// leaves empty, or `None` when the permission is not set there. +fn drop_owned_permission(text: &str, file: &str) -> Option { + let body = text.trim_start_matches('\u{feff}'); + let bom = &text[..text.len() - body.len()]; + let mut document = body.parse::().ok()?; + let keys = hatch::permission_keys(file == hatch::HATCH_FILES[1]); + hatch::drop_direct_reference_permission(&mut document, keys) + .then(|| crate::utils::python_lock::preserve_line_endings(text, format!("{bom}{document}"))) +} + #[cfg(test)] mod tests { use super::*; @@ -477,6 +519,233 @@ mod tests { ); } + /// #674: a hosted→vendored takeover reverts one hosted pin at a time, + /// so the first package's ledger snapshots the permission the OTHER + /// package's still-live hosted reference needs. Once the last project + /// direct reference is unwired, in either order, the pyproject is back + /// to its pre-hosted bytes. + #[tokio::test] + async fn takeover_snapshot_of_hosted_permission_is_not_restored() { + let original = "[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[project]\nname = \"app\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\", \"toml==0.10.2\"]\n"; + let packages = [ + ("six", "1.16.0", UUID), + ("toml", "0.10.2", "a0f74f9a-ce65-4451-ab60-025159b4d410"), + ]; + let hosted_url = |name: &str, version: &str| { + format!( + "https://patches.example/{name}-{version}-py3-none-any.whl#sha256={}", + "b".repeat(64) + ) + }; + for order in [[0, 1], [1, 0]] { + for external in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let mut files = + BTreeMap::from([("pyproject.toml".to_owned(), original.to_owned())]); + if external { + files.insert( + "hatch.toml".into(), + "[metadata]\nallow-ambiguous-features = true\n".into(), + ); + } + let before = files.clone(); + // Hosted scan: both packages redirected, permission enabled. + for (name, version, _) in packages { + let plan = + hatch::plan(&files, name, version, &hosted_url(name, version)).unwrap(); + files.extend(plan.files); + } + let permission_file = if external { + "hatch.toml" + } else { + "pyproject.toml" + }; + assert!(files[permission_file].contains("allow-direct-references = true")); + for (file, text) in &files { + tokio::fs::write(root.join(file), text).await.unwrap(); + } + // Takeover: unwind one hosted pin (the other keeps the + // permission live), then vendor that package. + let mut state = VendorState::default(); + let mut entries = Vec::new(); + for (name, version, uuid) in packages { + let text = tokio::fs::read_to_string(root.join("pyproject.toml")) + .await + .unwrap(); + let unwound = text.replace( + &format!("{name} @ {}", hosted_url(name, version)), + &format!("{name}=={version}"), + ); + assert_ne!(unwound, text); + tokio::fs::write(root.join("pyproject.toml"), unwound) + .await + .unwrap(); + let project = load(root, name, version, uuid).await.unwrap(); + let wheel = + format!(".socket/vendor/pypi/{uuid}/{name}-{version}-py3-none-any.whl"); + let wiring = wire(&project, root, name, version, &wheel, &"0".repeat(64)) + .await + .unwrap(); + let entry = entry(uuid, name, &wheel, &"0".repeat(64), wiring); + state.entries.insert(name.into(), entry.clone()); + entries.push(entry); + save_state(root, &state).await.unwrap(); + } + for index in order { + let outcome = revert(&entries[index], root, false).await; + assert!(outcome.success, "{:?}", outcome.error); + } + for (file, text) in &before { + assert_eq!( + &tokio::fs::read_to_string(root.join(file)).await.unwrap(), + text, + "{file}, order {order:?}, external {external}" + ); + } + } + } + } + + /// #674 end to end through the real lanes: the hosted rewrite, then the + /// takeover's per-pin `restore_upstream` and vendored wiring for each + /// package in turn, then a revert in either order. + #[tokio::test] + async fn takeover_through_hosted_unwind_reverts_byte_exact() { + use crate::patch::redirect::upstream::{restore_upstream, HostedPin, RestoreOptions}; + let original = "[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[project]\nname = \"app\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\", \"toml==0.10.2\"]\n"; + let packages = [ + ("six", "1.16.0", UUID), + ("toml", "0.10.2", "a0f74f9a-ce65-4451-ab60-025159b4d410"), + ]; + let deps: Vec = packages + .iter() + .map(|(name, version, uuid)| { + serde_json::from_value(json!({ + "ecosystem": "pypi", "name": name, "version": version, + "token": "11111111-1111-1111-1111-111111111111", + "patchUuid": uuid, + "artifactUrl": format!( + "https://patch.socket.dev/patch/pypi/{name}/{version}/11111111-1111-1111-1111-111111111111/{uuid}/{name}-{version}-py3-none-any.whl" + ), + "integrity": { "sha256": "d".repeat(64) } + })) + .unwrap() + }) + .collect(); + let input = BTreeMap::from([("pyproject.toml".to_owned(), original.to_owned())]); + let hosted = crate::patch::redirect::rewrite_registry_redirect_with_pipenv_version( + &input, + &deps, + &BTreeMap::new(), + None, + false, + ); + let hosted = &hosted.files["pyproject.toml"]; + assert!( + hosted.contains("allow-direct-references = true"), + "{hosted}" + ); + for order in [[0, 1], [1, 0]] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + tokio::fs::write(root.join("pyproject.toml"), hosted) + .await + .unwrap(); + let pins = HostedPin::all(&crate::vex::discover_patched_refs(root).await); + assert_eq!(pins.len(), 2, "{pins:?}"); + let mut state = VendorState::default(); + let mut entries = Vec::new(); + for (name, version, uuid) in packages { + let pin = pins.iter().find(|pin| pin.uuid == uuid).unwrap(); + let restore = restore_upstream( + root, + std::slice::from_ref(pin), + &RestoreOptions { + offline: true, + ..RestoreOptions::default() + }, + ) + .await; + assert_eq!(restore.refused().count(), 0, "{name}"); + let project = load(root, name, version, uuid).await.unwrap(); + let wheel = format!(".socket/vendor/pypi/{uuid}/{name}-{version}-py3-none-any.whl"); + let wiring = wire(&project, root, name, version, &wheel, &"0".repeat(64)) + .await + .unwrap(); + let entry = entry(uuid, name, &wheel, &"0".repeat(64), wiring); + state.entries.insert(name.into(), entry.clone()); + entries.push(entry); + save_state(root, &state).await.unwrap(); + } + for index in order { + let outcome = revert(&entries[index], root, false).await; + assert!(outcome.success, "{:?}", outcome.error); + } + assert_eq!( + tokio::fs::read_to_string(root.join("pyproject.toml")) + .await + .unwrap(), + original, + "order {order:?}" + ); + } + } + + /// The #674 drop only applies to a permission recorded while + /// non-vendored direct references were live. A permission the user set + /// is restored verbatim after two vendored packages are reverted, inline + /// and in hatch.toml alike. + #[tokio::test] + async fn user_permission_survives_vendored_revert() { + let pyproject = "[project]\nname = \"app\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\", \"toml==0.10.2\"]\n"; + for (file, text) in [ + ( + "pyproject.toml", + format!("{pyproject}\n[tool.hatch.metadata]\nallow-direct-references = true\n"), + ), + ( + "hatch.toml", + "[metadata]\nallow-direct-references = true\n".to_owned(), + ), + ] { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let mut before = BTreeMap::from([("pyproject.toml".to_owned(), pyproject.to_owned())]); + before.insert(file.to_owned(), text); + for (file, text) in &before { + tokio::fs::write(root.join(file), text).await.unwrap(); + } + let mut state = VendorState::default(); + let mut entries = Vec::new(); + for (name, version, uuid) in [ + ("six", "1.16.0", UUID), + ("toml", "0.10.2", "a0f74f9a-ce65-4451-ab60-025159b4d410"), + ] { + let project = load(root, name, version, uuid).await.unwrap(); + let wheel = format!(".socket/vendor/pypi/{uuid}/{name}-{version}-py3-none-any.whl"); + let wiring = wire(&project, root, name, version, &wheel, &"0".repeat(64)) + .await + .unwrap(); + let entry = entry(uuid, name, &wheel, &"0".repeat(64), wiring); + state.entries.insert(name.into(), entry.clone()); + entries.push(entry); + save_state(root, &state).await.unwrap(); + } + for entry in entries.iter().rev() { + let outcome = revert(entry, root, false).await; + assert!(outcome.success, "{:?}", outcome.error); + } + for (name, text) in &before { + assert_eq!( + &tokio::fs::read_to_string(root.join(name)).await.unwrap(), + text, + "{name} ({file} permission)" + ); + } + } + } + /// #385: ordinary pyproject edits after vendoring (a release bump, a /// comment on `name`, a new sibling dependency) must not block rollback. #[tokio::test]